From 193fc241ca125df703d62ae87da7303f67213e38 Mon Sep 17 00:00:00 2001
From: Guy Benoish <guy.benoish@redislabs.com>
Date: Tue, 21 Jan 2020 15:09:42 +0530
Subject: [PATCH 001/186] Fix memory corruption in moduleHandleBlockedClients

By using a "circular BRPOPLPUSH"-like scenario it was
possible the get the same client on db->blocking_keys
twice (See comment in moduleTryServeClientBlockedOnKey)

The fix was actually already implememnted in
moduleTryServeClientBlockedOnKey but it had a bug:
the funxction should return 0 or 1 (not OK or ERR)

Other changes:
1. Added two commands to blockonkeys.c test module (To
   reproduce the case described above)
2. Simplify blockonkeys.c in order to make testing easier
3. cast raxSize() to avoid warning with format spec
---
 src/module.c                         |  18 +++-
 tests/modules/blockonkeys.c          | 122 ++++++++++++++++++++++-----
 tests/unit/moduleapi/blockonkeys.tcl |  55 +++++++-----
 3 files changed, 149 insertions(+), 46 deletions(-)

diff --git a/src/module.c b/src/module.c
index 61dc25169..85e1497fd 100644
--- a/src/module.c
+++ b/src/module.c
@@ -4393,14 +4393,26 @@ RedisModuleBlockedClient *moduleBlockClient(RedisModuleCtx *ctx, RedisModuleCmdF
  * can really be unblocked, since the module was able to serve the client.
  * If the callback returns REDISMODULE_OK, then the client can be unblocked,
  * otherwise the client remains blocked and we'll retry again when one of
- * the keys it blocked for becomes "ready" again. */
+ * the keys it blocked for becomes "ready" again.
+ * This function returns 1 if client was served (and should be unblocked) */
 int moduleTryServeClientBlockedOnKey(client *c, robj *key) {
     int served = 0;
     RedisModuleBlockedClient *bc = c->bpop.module_blocked_handle;
     /* Protect against re-processing: don't serve clients that are already
      * in the unblocking list for any reason (including RM_UnblockClient()
-     * explicit call). */
-    if (bc->unblocked) return REDISMODULE_ERR;
+     * explicit call).
+     * For example, the following pathological case:
+     * Assume a module called LIST implements the same command as
+     * the Redis list data type.
+     * LIST.BRPOPLPUSH src dst 0 ('src' goes into db->blocking_keys)
+     * LIST.BRPOPLPUSH dst src 0 ('dst' goes into db->blocking_keys)
+     * LIST.LPUSH src foo
+     * 'src' is in db->blocking_keys after the first BRPOPLPUSH is served
+     * (and stays there until the next beforeSleep).
+     * The second BRPOPLPUSH will signal 'src' as ready, leading to the
+     * unblocking of the already unblocked (and worst, freed) reply_client
+     * of the first BRPOPLPUSH. */
+    if (bc->unblocked) return 0;
     RedisModuleCtx ctx = REDISMODULE_CTX_INIT;
     ctx.flags |= REDISMODULE_CTX_BLOCKED_REPLY;
     ctx.blocked_ready_key = key;
diff --git a/tests/modules/blockonkeys.c b/tests/modules/blockonkeys.c
index 10dc65b1a..94f31d455 100644
--- a/tests/modules/blockonkeys.c
+++ b/tests/modules/blockonkeys.c
@@ -109,41 +109,33 @@ int fsl_push(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
         return RedisModule_ReplyWithError(ctx,"ERR new element has to be greater than the head element");
 
     fsl->list[fsl->length++] = ele;
-
-    if (fsl->length >= 2)
-        RedisModule_SignalKeyAsReady(ctx, argv[1]);
+    RedisModule_SignalKeyAsReady(ctx, argv[1]);
 
     return RedisModule_ReplyWithSimpleString(ctx, "OK");
 }
 
-int bpop2_reply_callback(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
+int bpop_reply_callback(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
     REDISMODULE_NOT_USED(argv);
     REDISMODULE_NOT_USED(argc);
     RedisModuleString *keyname = RedisModule_GetBlockedClientReadyKey(ctx);
 
     fsl_t *fsl;
-    if (!get_fsl(ctx, keyname, REDISMODULE_READ, 0, &fsl, 0))
+    if (!get_fsl(ctx, keyname, REDISMODULE_READ, 0, &fsl, 0) || !fsl)
         return REDISMODULE_ERR;
 
-    if (!fsl || fsl->length < 2)
-        return REDISMODULE_ERR;
-
-    RedisModule_ReplyWithArray(ctx, 2);
-    RedisModule_ReplyWithLongLong(ctx, fsl->list[--fsl->length]);
     RedisModule_ReplyWithLongLong(ctx, fsl->list[--fsl->length]);
     return REDISMODULE_OK;
 }
 
-int bpop2_timeout_callback(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
+int bpop_timeout_callback(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
     REDISMODULE_NOT_USED(argv);
     REDISMODULE_NOT_USED(argc);
     return RedisModule_ReplyWithSimpleString(ctx, "Request timedout");
 }
 
-
-/* FSL.BPOP2 <key> <timeout> - Block clients until list has two or more elements.
+/* FSL.BPOP <key> <timeout> - Block clients until list has two or more elements.
  * When that happens, unblock client and pop the last two elements (from the right). */
-int fsl_bpop2(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
+int fsl_bpop(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
     if (argc != 3)
         return RedisModule_WrongArity(ctx);
 
@@ -155,13 +147,10 @@ int fsl_bpop2(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
     if (!get_fsl(ctx, argv[1], REDISMODULE_READ, 0, &fsl, 1))
         return REDISMODULE_OK;
 
-    if (!fsl || fsl->length < 2) {
-        /* Key is empty or has <2 elements, we must block */
-        RedisModule_BlockClientOnKeys(ctx, bpop2_reply_callback, bpop2_timeout_callback,
+    if (!fsl) {
+        RedisModule_BlockClientOnKeys(ctx, bpop_reply_callback, bpop_timeout_callback,
                                       NULL, timeout, &argv[1], 1, NULL);
     } else {
-        RedisModule_ReplyWithArray(ctx, 2);
-        RedisModule_ReplyWithLongLong(ctx, fsl->list[--fsl->length]);
         RedisModule_ReplyWithLongLong(ctx, fsl->list[--fsl->length]);
     }
 
@@ -175,10 +164,10 @@ int bpopgt_reply_callback(RedisModuleCtx *ctx, RedisModuleString **argv, int arg
     long long *pgt = RedisModule_GetBlockedClientPrivateData(ctx);
 
     fsl_t *fsl;
-    if (!get_fsl(ctx, keyname, REDISMODULE_READ, 0, &fsl, 0))
+    if (!get_fsl(ctx, keyname, REDISMODULE_READ, 0, &fsl, 0) || !fsl)
         return REDISMODULE_ERR;
 
-    if (!fsl || fsl->list[fsl->length-1] <= *pgt)
+    if (fsl->list[fsl->length-1] <= *pgt)
         return REDISMODULE_ERR;
 
     RedisModule_ReplyWithLongLong(ctx, fsl->list[--fsl->length]);
@@ -218,7 +207,6 @@ int fsl_bpopgt(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
         /* We use malloc so the tests in blockedonkeys.tcl can check for memory leaks */
         long long *pgt = RedisModule_Alloc(sizeof(long long));
         *pgt = gt;
-        /* Key is empty or has <2 elements, we must block */
         RedisModule_BlockClientOnKeys(ctx, bpopgt_reply_callback, bpopgt_timeout_callback,
                                       bpopgt_free_privdata, timeout, &argv[1], 1, pgt);
     } else {
@@ -228,6 +216,88 @@ int fsl_bpopgt(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
     return REDISMODULE_OK;
 }
 
+int bpoppush_reply_callback(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
+    REDISMODULE_NOT_USED(argv);
+    REDISMODULE_NOT_USED(argc);
+    RedisModuleString *src_keyname = RedisModule_GetBlockedClientReadyKey(ctx);
+    RedisModuleString *dst_keyname = RedisModule_GetBlockedClientPrivateData(ctx);
+
+    fsl_t *src;
+    if (!get_fsl(ctx, src_keyname, REDISMODULE_READ, 0, &src, 0) || !src)
+        return REDISMODULE_ERR;
+
+    fsl_t *dst;
+    if (!get_fsl(ctx, dst_keyname, REDISMODULE_WRITE, 1, &dst, 0) || !dst)
+        return REDISMODULE_ERR;
+
+    long long ele = src->list[--src->length];
+    dst->list[dst->length++] = ele;
+    RedisModule_SignalKeyAsReady(ctx, dst_keyname);
+    return RedisModule_ReplyWithLongLong(ctx, ele);
+}
+
+int bpoppush_timeout_callback(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
+    REDISMODULE_NOT_USED(argv);
+    REDISMODULE_NOT_USED(argc);
+    return RedisModule_ReplyWithSimpleString(ctx, "Request timedout");
+}
+
+void bpoppush_free_privdata(RedisModuleCtx *ctx, void *privdata) {
+    RedisModule_FreeString(ctx, privdata);
+}
+
+/* FSL.BPOPPUSH <src> <dst> <timeout> - Block clients until <src> has an element.
+ * When that happens, unblock client, pop the last element from <src> and push it to <dst>
+ * (from the right). */
+int fsl_bpoppush(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
+    if (argc != 4)
+        return RedisModule_WrongArity(ctx);
+
+    long long timeout;
+    if (RedisModule_StringToLongLong(argv[3],&timeout) != REDISMODULE_OK || timeout < 0)
+        return RedisModule_ReplyWithError(ctx,"ERR invalid timeout");
+
+    fsl_t *src;
+    if (!get_fsl(ctx, argv[1], REDISMODULE_READ, 0, &src, 1))
+        return REDISMODULE_OK;
+
+    if (!src) {
+        /* Retain string for reply callback */
+        RedisModule_RetainString(ctx, argv[2]);
+        /* Key is empty, we must block */
+        RedisModule_BlockClientOnKeys(ctx, bpoppush_reply_callback, bpoppush_timeout_callback,
+                                      bpoppush_free_privdata, timeout, &argv[1], 1, argv[2]);
+    } else {
+        fsl_t *dst;
+        if (!get_fsl(ctx, argv[2], REDISMODULE_WRITE, 1, &dst, 1))
+            return REDISMODULE_OK;
+        long long ele = src->list[--src->length];
+        dst->list[dst->length++] = ele;
+        RedisModule_SignalKeyAsReady(ctx, argv[2]);
+        RedisModule_ReplyWithLongLong(ctx, ele);
+    }
+
+    return REDISMODULE_OK;
+}
+
+/* FSL.GETALL <key> - Reply with an array containing all elements. */
+int fsl_getall(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
+    if (argc != 2)
+        return RedisModule_WrongArity(ctx);
+
+    fsl_t *fsl;
+    if (!get_fsl(ctx, argv[1], REDISMODULE_READ, 0, &fsl, 1))
+        return REDISMODULE_OK;
+
+    if (!fsl)
+        return RedisModule_ReplyWithArray(ctx, 0);
+
+    RedisModule_ReplyWithArray(ctx, fsl->length);
+    for (int i = 0; i < fsl->length; i++)
+        RedisModule_ReplyWithLongLong(ctx, fsl->list[i]);
+    return REDISMODULE_OK;
+}
+
 int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
     REDISMODULE_NOT_USED(argv);
     REDISMODULE_NOT_USED(argc);
@@ -252,11 +322,17 @@ int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc)
     if (RedisModule_CreateCommand(ctx,"fsl.push",fsl_push,"",0,0,0) == REDISMODULE_ERR)
         return REDISMODULE_ERR;
 
-    if (RedisModule_CreateCommand(ctx,"fsl.bpop2",fsl_bpop2,"",0,0,0) == REDISMODULE_ERR)
+    if (RedisModule_CreateCommand(ctx,"fsl.bpop",fsl_bpop,"",0,0,0) == REDISMODULE_ERR)
         return REDISMODULE_ERR;
 
     if (RedisModule_CreateCommand(ctx,"fsl.bpopgt",fsl_bpopgt,"",0,0,0) == REDISMODULE_ERR)
         return REDISMODULE_ERR;
 
+    if (RedisModule_CreateCommand(ctx,"fsl.bpoppush",fsl_bpoppush,"",0,0,0) == REDISMODULE_ERR)
+        return REDISMODULE_ERR;
+
+    if (RedisModule_CreateCommand(ctx,"fsl.getall",fsl_getall,"",0,0,0) == REDISMODULE_ERR)
+        return REDISMODULE_ERR;
+
     return REDISMODULE_OK;
 }
diff --git a/tests/unit/moduleapi/blockonkeys.tcl b/tests/unit/moduleapi/blockonkeys.tcl
index b380227e0..c8b8f23ed 100644
--- a/tests/unit/moduleapi/blockonkeys.tcl
+++ b/tests/unit/moduleapi/blockonkeys.tcl
@@ -3,37 +3,53 @@ set testmodule [file normalize tests/modules/blockonkeys.so]
 start_server {tags {"modules"}} {
     r module load $testmodule
 
+    test "Module client blocked on keys: Circular BPOPPUSH" {
+        set rd1 [redis_deferring_client]
+        set rd2 [redis_deferring_client]
+
+        r del src dst
+
+        $rd1 fsl.bpoppush src dst 0
+        $rd2 fsl.bpoppush dst src 0
+
+        r fsl.push src 42
+
+        assert_equal {42} [r fsl.getall src]
+        assert_equal {} [r fsl.getall dst]
+    }
+
+    test "Module client blocked on keys: Self-referential BPOPPUSH" {
+        set rd1 [redis_deferring_client]
+
+        r del src
+
+        $rd1 fsl.bpoppush src src 0
+
+        r fsl.push src 42
+
+        assert_equal {42} [r fsl.getall src]
+    }
+
     test {Module client blocked on keys (no metadata): No block} {
         r del k
         r fsl.push k 33
         r fsl.push k 34
-        r fsl.bpop2 k 0
-    } {34 33}
+        r fsl.bpop k 0
+    } {34}
 
     test {Module client blocked on keys (no metadata): Timeout} {
         r del k
         set rd [redis_deferring_client]
-        r fsl.push k 33
-        $rd fsl.bpop2 k 1
+        $rd fsl.bpop k 1
         assert_equal {Request timedout} [$rd read]
     }
 
-    test {Module client blocked on keys (no metadata): Blocked, case 1} {
+    test {Module client blocked on keys (no metadata): Blocked} {
         r del k
         set rd [redis_deferring_client]
-        r fsl.push k 33
-        $rd fsl.bpop2 k 0
+        $rd fsl.bpop k 0
         r fsl.push k 34
-        assert_equal {34 33} [$rd read]
-    }
-
-    test {Module client blocked on keys (no metadata): Blocked, case 2} {
-        r del k
-        set rd [redis_deferring_client]
-        r fsl.push k 33
-        r fsl.push k 34
-        $rd fsl.bpop2 k 0
-        assert_equal {34 33} [$rd read]
+        assert_equal {34} [$rd read]
     }
 
     test {Module client blocked on keys (with metadata): No block} {
@@ -108,13 +124,12 @@ start_server {tags {"modules"}} {
     test {Module client blocked on keys does not wake up on wrong type} {
         r del k
         set rd [redis_deferring_client]
-        $rd fsl.bpop2 k 0
+        $rd fsl.bpop k 0
         r lpush k 12
         r lpush k 13
         r lpush k 14
         r del k
-        r fsl.push k 33
         r fsl.push k 34
-        assert_equal {34 33} [$rd read]
+        assert_equal {34} [$rd read]
     }
 }

From 1907e0f18faa29c28df57d9d21982c5d8a942013 Mon Sep 17 00:00:00 2001
From: Guy Benoish <guy.benoish@redislabs.com>
Date: Sun, 29 Mar 2020 17:50:42 +0300
Subject: [PATCH 002/186] PERSIST should notify a keyspace event

---
 src/expire.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/expire.c b/src/expire.c
index 5aff72ee0..c102a01ff 100644
--- a/src/expire.c
+++ b/src/expire.c
@@ -590,6 +590,7 @@ void pttlCommand(client *c) {
 void persistCommand(client *c) {
     if (lookupKeyWrite(c->db,c->argv[1])) {
         if (removeExpire(c->db,c->argv[1])) {
+            notifyKeyspaceEvent(NOTIFY_GENERIC,"persist",c->argv[1],c->db->id);
             addReply(c,shared.cone);
             server.dirty++;
         } else {

From 4395889c9e13b0f0909372780fd1d6ecad7fb61a Mon Sep 17 00:00:00 2001
From: hwware <wen.hui.ware@gmail.com>
Date: Sun, 29 Mar 2020 23:06:50 -0400
Subject: [PATCH 003/186] add check for not providing both optin optout flag

---
 src/networking.c | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/src/networking.c b/src/networking.c
index 3c754c376..f8808f08f 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -2326,6 +2326,14 @@ NULL
                 return;
             }
 
+            if ((options & CLIENT_TRACKING_OPTIN) && (options & CLIENT_TRACKING_OPTOUT))
+            {
+                addReplyError(c,
+                "You can't specify both OPTIN mode and OPTOUT mode");
+                zfree(prefix);
+                return;
+            }
+
             enableTracking(c,redir,options,prefix,numprefix);
         } else if (!strcasecmp(c->argv[2]->ptr,"off")) {
             disableTracking(c);

From b35407fa7c5e43ce184c89abb50134e816292907 Mon Sep 17 00:00:00 2001
From: hwware <wen.hui.ware@gmail.com>
Date: Sun, 29 Mar 2020 23:20:54 -0400
Subject: [PATCH 004/186] add check for not switching between optin optout mode
 directly

---
 src/networking.c | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/src/networking.c b/src/networking.c
index f8808f08f..85c640e34 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -2326,7 +2326,7 @@ NULL
                 return;
             }
 
-            if ((options & CLIENT_TRACKING_OPTIN) && (options & CLIENT_TRACKING_OPTOUT))
+            if (options & CLIENT_TRACKING_OPTIN && options & CLIENT_TRACKING_OPTOUT)
             {
                 addReplyError(c,
                 "You can't specify both OPTIN mode and OPTOUT mode");
@@ -2334,6 +2334,17 @@ NULL
                 return;
             }
 
+            if ((options & CLIENT_TRACKING_OPTIN && c->flags & CLIENT_TRACKING_OPTOUT) ||
+                (options & CLIENT_TRACKING_OPTOUT && c->flags & CLIENT_TRACKING_OPTIN))
+            {
+                addReplyError(c,
+                "You can't switch OPTIN/OPTOUT mode before disabling "
+                "tracking for this client, and then re-enabling it with "
+                "a different mode.");
+                zfree(prefix);
+                return;
+            }
+
             enableTracking(c,redir,options,prefix,numprefix);
         } else if (!strcasecmp(c->argv[2]->ptr,"off")) {
             disableTracking(c);

From 7764996becceb9d363a838e69fc781a24fae02b9 Mon Sep 17 00:00:00 2001
From: Guy Benoish <guy.benoish@redislabs.com>
Date: Tue, 5 Nov 2019 19:23:37 +0530
Subject: [PATCH 005/186] Make sure Redis does not reply with negative zero

---
 src/util.c               | 4 ++++
 tests/unit/type/incr.tcl | 6 ++++++
 2 files changed, 10 insertions(+)

diff --git a/src/util.c b/src/util.c
index 2be42a0df..bd8f0fb98 100644
--- a/src/util.c
+++ b/src/util.c
@@ -602,6 +602,10 @@ int ld2string(char *buf, size_t len, long double value, ld2string_mode mode) {
                 }
                 if (*p == '.') l--;
             }
+            if (l == 2 && buf[0] == '-' && buf[1] == '0') {
+                buf[0] = '0';
+                l = 1;
+            }
             break;
         default: return 0; /* Invalid mode. */
         }
diff --git a/tests/unit/type/incr.tcl b/tests/unit/type/incr.tcl
index a58710d39..63bf2e116 100644
--- a/tests/unit/type/incr.tcl
+++ b/tests/unit/type/incr.tcl
@@ -151,4 +151,10 @@ start_server {tags {"incr"}} {
         catch {r incrbyfloat foo 1} err
         format $err
     } {ERR*valid*}
+
+    test {No negative zero} {
+        r incrbyfloat foo [expr double(1)/41]
+        r incrbyfloat foo [expr double(-1)/41]
+        r get foo
+    } {0}
 }

From 209f3a1ebac8d9f74846ccede58a70bc945ced0f Mon Sep 17 00:00:00 2001
From: Xudong Zhang <zxdvd_zhang@163.com>
Date: Thu, 2 Apr 2020 23:43:19 +0800
Subject: [PATCH 006/186] fix integer overflow

---
 src/quicklist.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/quicklist.c b/src/quicklist.c
index ae183ffd8..52e3988f5 100644
--- a/src/quicklist.c
+++ b/src/quicklist.c
@@ -110,7 +110,7 @@ quicklist *quicklistCreate(void) {
     return quicklist;
 }
 
-#define COMPRESS_MAX (1 << QL_COMP_BITS)
+#define COMPRESS_MAX ((1 << QL_COMP_BITS)-1)
 void quicklistSetCompressDepth(quicklist *quicklist, int compress) {
     if (compress > COMPRESS_MAX) {
         compress = COMPRESS_MAX;
@@ -120,7 +120,7 @@ void quicklistSetCompressDepth(quicklist *quicklist, int compress) {
     quicklist->compress = compress;
 }
 
-#define FILL_MAX (1 << (QL_FILL_BITS-1))
+#define FILL_MAX ((1 << (QL_FILL_BITS-1))-1)
 void quicklistSetFill(quicklist *quicklist, int fill) {
     if (fill > FILL_MAX) {
         fill = FILL_MAX;

From 8cdc153f58e535eff4be5f8f5483794c4cfeec3c Mon Sep 17 00:00:00 2001
From: Valentino Geron <valentino@redislabs.com>
Date: Wed, 25 Mar 2020 21:54:14 +0200
Subject: [PATCH 007/186] XACK should be executed in a "all or nothing"
 fashion.

First, we must parse the IDs, so that we abort ASAP.
The return value of this command cannot be an error if
the client successfully acknowledged some messages,
so it should be executed in a "all or nothing" fashion.
---
 src/t_stream.c                     | 12 +++++++++++-
 tests/unit/type/stream-cgroups.tcl | 12 ++++++++++++
 2 files changed, 23 insertions(+), 1 deletion(-)

diff --git a/src/t_stream.c b/src/t_stream.c
index 00d1cbf1c..3f8cbfcfa 100644
--- a/src/t_stream.c
+++ b/src/t_stream.c
@@ -1922,11 +1922,21 @@ void xackCommand(client *c) {
         return;
     }
 
+    /* Start parsing the IDs, so that we abort ASAP if there is a syntax
+     * error: the return value of this command cannot be an error in case
+     * the client successfully acknowledged some messages, so it should be
+     * executed in a "all or nothing" fashion. */
+    for (int j = 3; j < c->argc; j++) {
+        streamID id;
+        if (streamParseStrictIDOrReply(c,c->argv[j],&id,0) != C_OK) return;
+    }
+
     int acknowledged = 0;
     for (int j = 3; j < c->argc; j++) {
         streamID id;
         unsigned char buf[sizeof(streamID)];
-        if (streamParseStrictIDOrReply(c,c->argv[j],&id,0) != C_OK) return;
+        if (streamParseStrictIDOrReply(c,c->argv[j],&id,0) != C_OK)
+            serverPanic("StreamID invalid after check. Should not be possible.");
         streamEncodeID(buf,&id);
 
         /* Lookup the ID in the group PEL: it will have a reference to the
diff --git a/tests/unit/type/stream-cgroups.tcl b/tests/unit/type/stream-cgroups.tcl
index a27e1f582..04661707b 100644
--- a/tests/unit/type/stream-cgroups.tcl
+++ b/tests/unit/type/stream-cgroups.tcl
@@ -93,6 +93,18 @@ start_server {
         assert {[r XACK mystream mygroup $id1 $id2] eq 1}
     }
 
+    test {XACK should fail if got at least one invalid ID} {
+        r del mystream
+        r xgroup create s g $ MKSTREAM
+        r xadd s * f1 v1
+        set c [llength [lindex [r xreadgroup group g c streams s >] 0 1]]
+        assert {$c == 1}
+        set pending [r xpending s g - + 10 c]
+        set id1 [lindex $pending 0 0]
+        assert_error "*Invalid stream ID specified*" {r xack s g $id1 invalid-id}
+        assert {[r xack s g $id1] eq 1}
+    }
+
     test {PEL NACK reassignment after XGROUP SETID event} {
         r del events
         r xadd events * f1 v1

From 061616c1b1d022660f232a75d3ff2262b970a98d Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Tue, 10 Dec 2019 11:16:13 +0200
Subject: [PATCH 008/186] fix possible warning on incomplete struct init

---
 src/module.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/module.c b/src/module.c
index 16d24152e..64f274e78 100644
--- a/src/module.c
+++ b/src/module.c
@@ -5978,7 +5978,7 @@ sds modulesCollectInfo(sds info, const char *section, int for_crash_report, int
         struct RedisModule *module = dictGetVal(de);
         if (!module->info_cb)
             continue;
-        RedisModuleInfoCtx info_ctx = {module, section, info, sections, 0};
+        RedisModuleInfoCtx info_ctx = {module, section, info, sections, 0, 0};
         module->info_cb(&info_ctx, for_crash_report);
         /* Implicitly end dicts (no way to handle errors, and we must add the newline). */
         if (info_ctx.in_dict_field)

From 240094c9b097a180be6f2070ee47fac83a884d04 Mon Sep 17 00:00:00 2001
From: Guy Benoish <guy.benoish@redislabs.com>
Date: Tue, 25 Feb 2020 16:51:35 +0530
Subject: [PATCH 009/186] Stale replica should allow MULTI/EXEC

Example: Client uses a pipe to send the following to a
stale replica:

MULTI
.. do something ...
DISCARD

The replica will reply the MUTLI with -MASTERDOWN and
execute the rest of the commands... A client using a
pipe might not be aware that MULTI failed until it's
too late.

I can't think of a reason why MULTI/EXEC/DISCARD should
not be executed on stale replicas...

Also, enable MULTI/EXEC/DISCARD during loading
---
 src/server.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/server.c b/src/server.c
index c89e9c075..ff7a39df5 100644
--- a/src/server.c
+++ b/src/server.c
@@ -672,15 +672,15 @@ struct redisCommand redisCommandTable[] = {
      0,NULL,1,1,1,0,0,0},
 
     {"multi",multiCommand,1,
-     "no-script fast @transaction",
+     "no-script fast ok-loading ok-stale @transaction",
      0,NULL,0,0,0,0,0,0},
 
     {"exec",execCommand,1,
-     "no-script no-monitor no-slowlog @transaction",
+     "no-script no-monitor no-slowlog ok-loading ok-stale @transaction",
      0,NULL,0,0,0,0,0,0},
 
     {"discard",discardCommand,1,
-     "no-script fast @transaction",
+     "no-script fast ok-loading ok-stale @transaction",
      0,NULL,0,0,0,0,0,0},
 
     {"sync",syncCommand,1,

From cd2b5df9712c2a5956d288a762a89eb32f06bd1f Mon Sep 17 00:00:00 2001
From: hwware <wen.hui.ware@gmail.com>
Date: Wed, 18 Mar 2020 09:48:03 -0400
Subject: [PATCH 010/186] fix spelling in cluster.c

---
 src/cluster.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/cluster.c b/src/cluster.c
index a2e9ff5b6..385ff5763 100644
--- a/src/cluster.c
+++ b/src/cluster.c
@@ -933,7 +933,7 @@ int clusterAddNode(clusterNode *node) {
     return (retval == DICT_OK) ? C_OK : C_ERR;
 }
 
-/* Remove a node from the cluster. The functio performs the high level
+/* Remove a node from the cluster. The function performs the high level
  * cleanup, calling freeClusterNode() for the low level cleanup.
  * Here we do the following:
  *

From 814874d68b62e00f3996646ae64bb1927ff7995f Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Sun, 23 Feb 2020 16:51:27 +0200
Subject: [PATCH 011/186] change CI to build and run the module api tests

---
 .github/workflows/ci.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index cc4991606..3a81d1a08 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -13,6 +13,8 @@ jobs:
       run: |
         sudo apt-get install tcl8.5
         ./runtest --clients 2 --verbose
+    - name: module api test
+      run: ./runtest-moduleapi --clients 2 --verbose
 
   build-ubuntu-old:
     runs-on: ubuntu-16.04

From 3e0d209625ca729fbebb92a2768e258b0e76b181 Mon Sep 17 00:00:00 2001
From: Valentino Geron <valentino@redislabs.com>
Date: Thu, 26 Mar 2020 11:49:21 +0200
Subject: [PATCH 012/186] XREAD and XREADGROUP should not be allowed from
 scripts when BLOCK option is being used

---
 src/server.c             |  4 ++--
 src/t_stream.c           |  5 +++++
 tests/unit/scripting.tcl | 11 +++++++++++
 3 files changed, 18 insertions(+), 2 deletions(-)

diff --git a/src/server.c b/src/server.c
index ff7a39df5..9ebf0ee6b 100644
--- a/src/server.c
+++ b/src/server.c
@@ -947,11 +947,11 @@ struct redisCommand redisCommandTable[] = {
      0,NULL,1,1,1,0,0,0},
 
     {"xread",xreadCommand,-4,
-     "read-only no-script @stream @blocking",
+     "read-only @stream @blocking",
      0,xreadGetKeys,1,1,1,0,0,0},
 
     {"xreadgroup",xreadCommand,-7,
-     "write no-script @stream @blocking",
+     "write @stream @blocking",
      0,xreadGetKeys,1,1,1,0,0,0},
 
     {"xgroup",xgroupCommand,-2,
diff --git a/src/t_stream.c b/src/t_stream.c
index 3f8cbfcfa..e0af87f97 100644
--- a/src/t_stream.c
+++ b/src/t_stream.c
@@ -1374,6 +1374,11 @@ void xreadCommand(client *c) {
         int moreargs = c->argc-i-1;
         char *o = c->argv[i]->ptr;
         if (!strcasecmp(o,"BLOCK") && moreargs) {
+            if (c->flags & CLIENT_LUA) {
+                /* There is no sense to use BLOCK option within LUA */
+                addReplyErrorFormat(c, "%s command is not allowed with BLOCK option from scripts", (char *)c->argv[0]->ptr);
+                return;
+            }
             i++;
             if (getTimeoutFromObjectOrReply(c,c->argv[i],&timeout,
                 UNIT_MILLISECONDS) != C_OK) return;
diff --git a/tests/unit/scripting.tcl b/tests/unit/scripting.tcl
index fb36d0b80..8b364b287 100644
--- a/tests/unit/scripting.tcl
+++ b/tests/unit/scripting.tcl
@@ -146,6 +146,17 @@ start_server {tags {"scripting"}} {
         set e
     } {*not allowed*}
 
+    test {EVAL - Scripts can't run XREAD and XREADGROUP with BLOCK option} {
+        r del s
+        r xgroup create s g $ MKSTREAM
+        set res [r eval {return redis.pcall('xread','STREAMS','s','$')} 1 s]
+        assert {$res eq {}}
+        assert_error "*xread command is not allowed with BLOCK option from scripts" {r eval {return redis.pcall('xread','BLOCK',0,'STREAMS','s','$')} 1 s}
+        set res [r eval {return redis.pcall('xreadgroup','group','g','c','STREAMS','s','>')} 1 s]
+        assert {$res eq {}}
+        assert_error "*xreadgroup command is not allowed with BLOCK option from scripts" {r eval {return redis.pcall('xreadgroup','group','g','c','BLOCK',0,'STREAMS','s','>')} 1 s}
+    }
+
     test {EVAL - Scripts can't run certain commands} {
         set e {}
         r debug lua-always-replicate-commands 0

From 6fe66e096957f4ceb511629bf6452dcb27442567 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Thu, 2 Apr 2020 11:20:09 +0200
Subject: [PATCH 013/186] Simplify comment in
 moduleTryServeClientBlockedOnKey().

---
 src/module.c | 15 +++------------
 1 file changed, 3 insertions(+), 12 deletions(-)

diff --git a/src/module.c b/src/module.c
index 85e1497fd..16d24152e 100644
--- a/src/module.c
+++ b/src/module.c
@@ -4398,21 +4398,12 @@ RedisModuleBlockedClient *moduleBlockClient(RedisModuleCtx *ctx, RedisModuleCmdF
 int moduleTryServeClientBlockedOnKey(client *c, robj *key) {
     int served = 0;
     RedisModuleBlockedClient *bc = c->bpop.module_blocked_handle;
+
     /* Protect against re-processing: don't serve clients that are already
      * in the unblocking list for any reason (including RM_UnblockClient()
-     * explicit call).
-     * For example, the following pathological case:
-     * Assume a module called LIST implements the same command as
-     * the Redis list data type.
-     * LIST.BRPOPLPUSH src dst 0 ('src' goes into db->blocking_keys)
-     * LIST.BRPOPLPUSH dst src 0 ('dst' goes into db->blocking_keys)
-     * LIST.LPUSH src foo
-     * 'src' is in db->blocking_keys after the first BRPOPLPUSH is served
-     * (and stays there until the next beforeSleep).
-     * The second BRPOPLPUSH will signal 'src' as ready, leading to the
-     * unblocking of the already unblocked (and worst, freed) reply_client
-     * of the first BRPOPLPUSH. */
+     * explicit call). See #6798. */
     if (bc->unblocked) return 0;
+
     RedisModuleCtx ctx = REDISMODULE_CTX_INIT;
     ctx.flags |= REDISMODULE_CTX_BLOCKED_REPLY;
     ctx.blocked_ready_key = key;

From 15c9e79a7ddc00156ab67ba2afbd2fbf01c981ed Mon Sep 17 00:00:00 2001
From: David Carlier <devnexen@gmail.com>
Date: Sat, 7 Mar 2020 10:43:41 +0000
Subject: [PATCH 014/186] debug, dump registers on arm too.

---
 src/debug.c | 82 +++++++++++++++++++++++++++++++++++------------------
 1 file changed, 55 insertions(+), 27 deletions(-)

diff --git a/src/debug.c b/src/debug.c
index 83e5b6197..c7d5c5336 100644
--- a/src/debug.c
+++ b/src/debug.c
@@ -1047,6 +1047,61 @@ void logRegisters(ucontext_t *uc) {
         (unsigned long) uc->uc_mcontext.gregs[18]
     );
     logStackContent((void**)uc->uc_mcontext.gregs[15]);
+    #elif defined(__aarch64__) /* Linux AArch64 */
+    serverLog(LL_WARNING,
+	      "\n"
+	      "X18:%016lx X19:%016lx\nX20:%016lx X21:%016lx\n"
+	      "X22:%016lx X23:%016lx\nX24:%016lx X25:%016lx\n"
+	      "X26:%016lx X27:%016lx\nX28:%016lx X29:%016lx\n"
+	      "X30:%016lx\n"
+	      "pc:%016lx sp:%016lx\npstate:%016lx fault_address:%016lx\n",
+	      (unsigned long) uc->uc_mcontext.regs[18],
+	      (unsigned long) uc->uc_mcontext.regs[19],
+	      (unsigned long) uc->uc_mcontext.regs[20],
+	      (unsigned long) uc->uc_mcontext.regs[21],
+	      (unsigned long) uc->uc_mcontext.regs[22],
+	      (unsigned long) uc->uc_mcontext.regs[23],
+	      (unsigned long) uc->uc_mcontext.regs[24],
+	      (unsigned long) uc->uc_mcontext.regs[25],
+	      (unsigned long) uc->uc_mcontext.regs[26],
+	      (unsigned long) uc->uc_mcontext.regs[27],
+	      (unsigned long) uc->uc_mcontext.regs[28],
+	      (unsigned long) uc->uc_mcontext.regs[29],
+	      (unsigned long) uc->uc_mcontext.regs[30],
+	      (unsigned long) uc->uc_mcontext.pc,
+	      (unsigned long) uc->uc_mcontext.sp,
+	      (unsigned long) uc->uc_mcontext.pstate,
+	      (unsigned long) uc->uc_mcontext.fault_address
+		      );
+	      logStackContent((void**)uc->uc_mcontext.sp);
+    #elif defined(__arm__) /* Linux ARM */
+    serverLog(LL_WARNING,
+	      "\n"
+	      "R10:%016lx R9 :%016lx\nR8 :%016lx R7 :%016lx\n"
+	      "R6 :%016lx R5 :%016lx\nR4 :%016lx R3 :%016lx\n"
+	      "R2 :%016lx R1 :%016lx\nR0 :%016lx EC :%016lx\n"
+	      "fp: %016lx ip:%016lx\n",
+	      "pc:%016lx sp:%016lx\ncpsr:%016lx fault_address:%016lx\n",
+	      (unsigned long) uc->uc_mcontext.arm_r10,
+	      (unsigned long) uc->uc_mcontext.arm_r9,
+	      (unsigned long) uc->uc_mcontext.arm_r8,
+	      (unsigned long) uc->uc_mcontext.arm_r7,
+	      (unsigned long) uc->uc_mcontext.arm_r6,
+	      (unsigned long) uc->uc_mcontext.arm_r5,
+	      (unsigned long) uc->uc_mcontext.arm_r4,
+	      (unsigned long) uc->uc_mcontext.arm_r3,
+	      (unsigned long) uc->uc_mcontext.arm_r2,
+	      (unsigned long) uc->uc_mcontext.arm_r1,
+	      (unsigned long) uc->uc_mcontext.arm_r0,
+	      (unsigned long) uc->uc_mcontext.error_code,
+	      (unsigned long) uc->uc_mcontext.arm_fp,
+	      (unsigned long) uc->uc_mcontext.arm_ip,
+	      (unsigned long) uc->uc_mcontext.arm_pc,
+	      (unsigned long) uc->uc_mcontext.arm_sp,
+	      (unsigned long) uc->uc_mcontext.arm_cpsr,
+	      (unsigned long) uc->uc_mcontext.fault_address
+		      );
+	      logStackContent((void**)uc->uc_mcontext.arm_sp);
     #endif
 #elif defined(__FreeBSD__)
     #if defined(__x86_64__)
@@ -1187,33 +1242,6 @@ void logRegisters(ucontext_t *uc) {
         (unsigned long) uc->uc_mcontext.mc_cs
     );
     logStackContent((void**)uc->uc_mcontext.mc_rsp);
-#elif defined(__aarch64__) /* Linux AArch64 */
-    serverLog(LL_WARNING,
-	      "\n"
-	      "X18:%016lx X19:%016lx\nX20:%016lx X21:%016lx\n"
-	      "X22:%016lx X23:%016lx\nX24:%016lx X25:%016lx\n"
-	      "X26:%016lx X27:%016lx\nX28:%016lx X29:%016lx\n"
-	      "X30:%016lx\n"
-	      "pc:%016lx sp:%016lx\npstate:%016lx fault_address:%016lx\n",
-	      (unsigned long) uc->uc_mcontext.regs[18],
-	      (unsigned long) uc->uc_mcontext.regs[19],
-	      (unsigned long) uc->uc_mcontext.regs[20],
-	      (unsigned long) uc->uc_mcontext.regs[21],
-	      (unsigned long) uc->uc_mcontext.regs[22],
-	      (unsigned long) uc->uc_mcontext.regs[23],
-	      (unsigned long) uc->uc_mcontext.regs[24],
-	      (unsigned long) uc->uc_mcontext.regs[25],
-	      (unsigned long) uc->uc_mcontext.regs[26],
-	      (unsigned long) uc->uc_mcontext.regs[27],
-	      (unsigned long) uc->uc_mcontext.regs[28],
-	      (unsigned long) uc->uc_mcontext.regs[29],
-	      (unsigned long) uc->uc_mcontext.regs[30],
-	      (unsigned long) uc->uc_mcontext.pc,
-	      (unsigned long) uc->uc_mcontext.sp,
-	      (unsigned long) uc->uc_mcontext.pstate,
-	      (unsigned long) uc->uc_mcontext.fault_address
-		      );
-	      logStackContent((void**)uc->uc_mcontext.sp);
 #else
     serverLog(LL_WARNING,
         "  Dumping of registers not supported for this OS/arch");

From a38ff404bdc3ef05a4c06c38c4eb604ff2d03479 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Mon, 23 Dec 2019 10:15:52 +0200
Subject: [PATCH 015/186] modules don't signalModifiedKey in setKey() since
 that's done (optionally) in RM_CloseKey

---
 src/db.c       | 6 +++---
 src/module.c   | 6 +++---
 src/server.h   | 2 +-
 src/t_string.c | 2 +-
 4 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/src/db.c b/src/db.c
index 211bb978d..cadfeb77b 100644
--- a/src/db.c
+++ b/src/db.c
@@ -221,7 +221,7 @@ void dbOverwrite(redisDb *db, robj *key, robj *val) {
  *    unless 'keepttl' is true.
  *
  * All the new keys in the database should be created via this interface. */
-void genericSetKey(redisDb *db, robj *key, robj *val, int keepttl) {
+void genericSetKey(redisDb *db, robj *key, robj *val, int keepttl, int signal) {
     if (lookupKeyWrite(db,key) == NULL) {
         dbAdd(db,key,val);
     } else {
@@ -229,12 +229,12 @@ void genericSetKey(redisDb *db, robj *key, robj *val, int keepttl) {
     }
     incrRefCount(val);
     if (!keepttl) removeExpire(db,key);
-    signalModifiedKey(db,key);
+    if (signal) signalModifiedKey(db,key);
 }
 
 /* Common case for genericSetKey() where the TTL is not retained. */
 void setKey(redisDb *db, robj *key, robj *val) {
-    genericSetKey(db,key,val,0);
+    genericSetKey(db,key,val,0,1);
 }
 
 /* Return true if the specified key exists in the specified database.
diff --git a/src/module.c b/src/module.c
index 64f274e78..4d3d9e1af 100644
--- a/src/module.c
+++ b/src/module.c
@@ -2157,7 +2157,7 @@ RedisModuleString *RM_RandomKey(RedisModuleCtx *ctx) {
 int RM_StringSet(RedisModuleKey *key, RedisModuleString *str) {
     if (!(key->mode & REDISMODULE_WRITE) || key->iter) return REDISMODULE_ERR;
     RM_DeleteKey(key);
-    setKey(key->db,key->key,str);
+    genericSetKey(key->db,key->key,str,0,0);
     key->value = str;
     return REDISMODULE_OK;
 }
@@ -2237,7 +2237,7 @@ int RM_StringTruncate(RedisModuleKey *key, size_t newlen) {
     if (key->value == NULL) {
         /* Empty key: create it with the new size. */
         robj *o = createObject(OBJ_STRING,sdsnewlen(NULL, newlen));
-        setKey(key->db,key->key,o);
+        genericSetKey(key->db,key->key,o,0,0);
         key->value = o;
         decrRefCount(o);
     } else {
@@ -3625,7 +3625,7 @@ int RM_ModuleTypeSetValue(RedisModuleKey *key, moduleType *mt, void *value) {
     if (!(key->mode & REDISMODULE_WRITE) || key->iter) return REDISMODULE_ERR;
     RM_DeleteKey(key);
     robj *o = createModuleObject(mt,value);
-    setKey(key->db,key->key,o);
+    genericSetKey(key->db,key->key,o,0,0);
     decrRefCount(o);
     key->value = o;
     return REDISMODULE_OK;
diff --git a/src/server.h b/src/server.h
index c4db4278e..dbd72281f 100644
--- a/src/server.h
+++ b/src/server.h
@@ -2057,7 +2057,7 @@ int objectSetLRUOrLFU(robj *val, long long lfu_freq, long long lru_idle,
 #define LOOKUP_NOTOUCH (1<<0)
 void dbAdd(redisDb *db, robj *key, robj *val);
 void dbOverwrite(redisDb *db, robj *key, robj *val);
-void genericSetKey(redisDb *db, robj *key, robj *val, int keepttl);
+void genericSetKey(redisDb *db, robj *key, robj *val, int keepttl, int signal);
 void setKey(redisDb *db, robj *key, robj *val);
 int dbExists(redisDb *db, robj *key);
 robj *dbRandomKey(redisDb *db);
diff --git a/src/t_string.c b/src/t_string.c
index 8ccd69eb9..3174e9ccd 100644
--- a/src/t_string.c
+++ b/src/t_string.c
@@ -84,7 +84,7 @@ void setGenericCommand(client *c, int flags, robj *key, robj *val, robj *expire,
         addReply(c, abort_reply ? abort_reply : shared.null[c->resp]);
         return;
     }
-    genericSetKey(c->db,key,val,flags & OBJ_SET_KEEPTTL);
+    genericSetKey(c->db,key,val,flags & OBJ_SET_KEEPTTL,1);
     server.dirty++;
     if (expire) setExpire(c,c->db,key,mstime()+milliseconds);
     notifyKeyspaceEvent(NOTIFY_STRING,"set",key,c->db->id);

From 91ed9b3c47c46b71be32b24a93cd514f8147f997 Mon Sep 17 00:00:00 2001
From: Guy Benoish <guy.benoish@redislabs.com>
Date: Thu, 6 Feb 2020 15:13:52 +0530
Subject: [PATCH 016/186] Modules: Perform printf-like format checks in
 variadic API

---
 src/redismodule.h | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/redismodule.h b/src/redismodule.h
index d26c41456..754ee4fc6 100644
--- a/src/redismodule.h
+++ b/src/redismodule.h
@@ -470,7 +470,7 @@ RedisModuleString *REDISMODULE_API_FUNC(RedisModule_CreateStringFromLongLong)(Re
 RedisModuleString *REDISMODULE_API_FUNC(RedisModule_CreateStringFromDouble)(RedisModuleCtx *ctx, double d);
 RedisModuleString *REDISMODULE_API_FUNC(RedisModule_CreateStringFromLongDouble)(RedisModuleCtx *ctx, long double ld, int humanfriendly);
 RedisModuleString *REDISMODULE_API_FUNC(RedisModule_CreateStringFromString)(RedisModuleCtx *ctx, const RedisModuleString *str);
-RedisModuleString *REDISMODULE_API_FUNC(RedisModule_CreateStringPrintf)(RedisModuleCtx *ctx, const char *fmt, ...);
+RedisModuleString *REDISMODULE_API_FUNC(RedisModule_CreateStringPrintf)(RedisModuleCtx *ctx, const char *fmt, ...); __attribute__ ((format (printf, 2, 3)));
 void REDISMODULE_API_FUNC(RedisModule_FreeString)(RedisModuleCtx *ctx, RedisModuleString *str);
 const char *REDISMODULE_API_FUNC(RedisModule_StringPtrLen)(const RedisModuleString *str, size_t *len);
 int REDISMODULE_API_FUNC(RedisModule_ReplyWithError)(RedisModuleCtx *ctx, const char *err);
@@ -554,8 +554,8 @@ void REDISMODULE_API_FUNC(RedisModule_SaveLongDouble)(RedisModuleIO *io, long do
 long double REDISMODULE_API_FUNC(RedisModule_LoadLongDouble)(RedisModuleIO *io);
 void *REDISMODULE_API_FUNC(RedisModule_LoadDataTypeFromString)(const RedisModuleString *str, const RedisModuleType *mt);
 RedisModuleString *REDISMODULE_API_FUNC(RedisModule_SaveDataTypeToString)(RedisModuleCtx *ctx, void *data, const RedisModuleType *mt);
-void REDISMODULE_API_FUNC(RedisModule_Log)(RedisModuleCtx *ctx, const char *level, const char *fmt, ...);
-void REDISMODULE_API_FUNC(RedisModule_LogIOError)(RedisModuleIO *io, const char *levelstr, const char *fmt, ...);
+void REDISMODULE_API_FUNC(RedisModule_Log)(RedisModuleCtx *ctx, const char *level, const char *fmt, ...); __attribute__ ((format (printf, 3, 4)));
+void REDISMODULE_API_FUNC(RedisModule_LogIOError)(RedisModuleIO *io, const char *levelstr, const char *fmt, ...); __attribute__ ((format (printf, 3, 4)));
 void REDISMODULE_API_FUNC(RedisModule__Assert)(const char *estr, const char *file, int line);
 void REDISMODULE_API_FUNC(RedisModule_LatencyAddSample)(const char *event, mstime_t latency);
 int REDISMODULE_API_FUNC(RedisModule_StringAppendBuffer)(RedisModuleCtx *ctx, RedisModuleString *str, const char *buf, size_t len);

From c35a53169ffdbdf73f91224b6e62f6129fb9a838 Mon Sep 17 00:00:00 2001
From: Guy Benoish <guy.benoish@redislabs.com>
Date: Mon, 30 Mar 2020 10:52:59 +0300
Subject: [PATCH 017/186] streamReplyWithRange: Redundant XSETIDs to replica

propagate_last_id is declared outside of the loop but used
only from within the loop. Once it's '1' it will never go
back to '0' and will replicate XSETID even for IDs that
don't actually change the last_id.
While not a serious bug (XSETID always used group->last_id
so there's no risk), it does causes redundant traffic
between master and its replicas
---
 src/t_stream.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/t_stream.c b/src/t_stream.c
index 557d1d642..00d1cbf1c 100644
--- a/src/t_stream.c
+++ b/src/t_stream.c
@@ -935,7 +935,6 @@ size_t streamReplyWithRange(client *c, stream *s, streamID *start, streamID *end
     streamIterator si;
     int64_t numfields;
     streamID id;
-    int propagate_last_id = 0;
 
     /* If the client is asking for some history, we serve it using a
      * different function, so that we return entries *solely* from its
@@ -951,6 +950,8 @@ size_t streamReplyWithRange(client *c, stream *s, streamID *start, streamID *end
         arraylen_ptr = addReplyDeferredLen(c);
     streamIteratorStart(&si,s,start,end,rev);
     while(streamIteratorGetID(&si,&id,&numfields)) {
+        int propagate_last_id = 0;
+
         /* Update the group last_id if needed. */
         if (group && streamCompareID(&id,&group->last_id) > 0) {
             group->last_id = id;

From eba28e2cea0b2632cf751426ada02adf24f273db Mon Sep 17 00:00:00 2001
From: Guy Benoish <guy.benoish@redislabs.com>
Date: Thu, 30 Jan 2020 19:15:12 +0530
Subject: [PATCH 018/186] DEBUG OBJECT should pass keyname to module when
 loading

---
 src/debug.c | 2 +-
 src/rdb.c   | 4 ++--
 src/rdb.h   | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/debug.c b/src/debug.c
index c7d5c5336..baaaa2424 100644
--- a/src/debug.c
+++ b/src/debug.c
@@ -490,7 +490,7 @@ NULL
             "encoding:%s serializedlength:%zu "
             "lru:%d lru_seconds_idle:%llu%s",
             (void*)val, val->refcount,
-            strenc, rdbSavedObjectLen(val),
+            strenc, rdbSavedObjectLen(val, c->argv[2]),
             val->lru, estimateObjectIdleTime(val)/1000, extra);
     } else if (!strcasecmp(c->argv[1]->ptr,"sdslen") && c->argc == 3) {
         dictEntry *de;
diff --git a/src/rdb.c b/src/rdb.c
index 5d34f5a32..fc8911979 100644
--- a/src/rdb.c
+++ b/src/rdb.c
@@ -1002,8 +1002,8 @@ ssize_t rdbSaveObject(rio *rdb, robj *o, robj *key) {
  * the rdbSaveObject() function. Currently we use a trick to get
  * this length with very little changes to the code. In the future
  * we could switch to a faster solution. */
-size_t rdbSavedObjectLen(robj *o) {
-    ssize_t len = rdbSaveObject(NULL,o,NULL);
+size_t rdbSavedObjectLen(robj *o, robj *key) {
+    ssize_t len = rdbSaveObject(NULL,o,key);
     serverAssertWithInfo(NULL,o,len != -1);
     return len;
 }
diff --git a/src/rdb.h b/src/rdb.h
index 4229beea8..b276a978b 100644
--- a/src/rdb.h
+++ b/src/rdb.h
@@ -143,7 +143,7 @@ int rdbSaveToSlavesSockets(rdbSaveInfo *rsi);
 void rdbRemoveTempFile(pid_t childpid);
 int rdbSave(char *filename, rdbSaveInfo *rsi);
 ssize_t rdbSaveObject(rio *rdb, robj *o, robj *key);
-size_t rdbSavedObjectLen(robj *o);
+size_t rdbSavedObjectLen(robj *o, robj *key);
 robj *rdbLoadObject(int type, rio *rdb, robj *key);
 void backgroundSaveDoneHandler(int exitcode, int bysignal);
 int rdbSaveKeyValuePair(rio *rdb, robj *key, robj *val, long long expiretime);

From 024c380b9da02bc4112822c0f5f9ac1388b4205b Mon Sep 17 00:00:00 2001
From: Guy Benoish <guy.benoish@redislabs.com>
Date: Thu, 2 Apr 2020 18:41:29 +0300
Subject: [PATCH 019/186] Fix no-negative-zero test

---
 tests/unit/type/incr.tcl | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/unit/type/incr.tcl b/tests/unit/type/incr.tcl
index 63bf2e116..b7a135203 100644
--- a/tests/unit/type/incr.tcl
+++ b/tests/unit/type/incr.tcl
@@ -153,6 +153,7 @@ start_server {tags {"incr"}} {
     } {ERR*valid*}
 
     test {No negative zero} {
+        r del foo
         r incrbyfloat foo [expr double(1)/41]
         r incrbyfloat foo [expr double(-1)/41]
         r get foo

From 0e42cfc365cc2f15763d81052f85a2ce6e85bc6d Mon Sep 17 00:00:00 2001
From: Guy Benoish <guy.benoish@redislabs.com>
Date: Fri, 3 Apr 2020 14:49:40 +0300
Subject: [PATCH 020/186] Use __attribute__ only if __GNUC__ is defined

---
 src/redismodule.h | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/src/redismodule.h b/src/redismodule.h
index 754ee4fc6..23b4d26e0 100644
--- a/src/redismodule.h
+++ b/src/redismodule.h
@@ -470,7 +470,11 @@ RedisModuleString *REDISMODULE_API_FUNC(RedisModule_CreateStringFromLongLong)(Re
 RedisModuleString *REDISMODULE_API_FUNC(RedisModule_CreateStringFromDouble)(RedisModuleCtx *ctx, double d);
 RedisModuleString *REDISMODULE_API_FUNC(RedisModule_CreateStringFromLongDouble)(RedisModuleCtx *ctx, long double ld, int humanfriendly);
 RedisModuleString *REDISMODULE_API_FUNC(RedisModule_CreateStringFromString)(RedisModuleCtx *ctx, const RedisModuleString *str);
-RedisModuleString *REDISMODULE_API_FUNC(RedisModule_CreateStringPrintf)(RedisModuleCtx *ctx, const char *fmt, ...); __attribute__ ((format (printf, 2, 3)));
+#ifdef __GNUC__
+RedisModuleString *REDISMODULE_API_FUNC(RedisModule_CreateStringPrintf)(RedisModuleCtx *ctx, const char *fmt, ...) __attribute__ ((format (printf, 2, 3)));
+#else
+RedisModuleString *REDISMODULE_API_FUNC(RedisModule_CreateStringPrintf)(RedisModuleCtx *ctx, const char *fmt, ...);
+#endif
 void REDISMODULE_API_FUNC(RedisModule_FreeString)(RedisModuleCtx *ctx, RedisModuleString *str);
 const char *REDISMODULE_API_FUNC(RedisModule_StringPtrLen)(const RedisModuleString *str, size_t *len);
 int REDISMODULE_API_FUNC(RedisModule_ReplyWithError)(RedisModuleCtx *ctx, const char *err);
@@ -554,8 +558,13 @@ void REDISMODULE_API_FUNC(RedisModule_SaveLongDouble)(RedisModuleIO *io, long do
 long double REDISMODULE_API_FUNC(RedisModule_LoadLongDouble)(RedisModuleIO *io);
 void *REDISMODULE_API_FUNC(RedisModule_LoadDataTypeFromString)(const RedisModuleString *str, const RedisModuleType *mt);
 RedisModuleString *REDISMODULE_API_FUNC(RedisModule_SaveDataTypeToString)(RedisModuleCtx *ctx, void *data, const RedisModuleType *mt);
-void REDISMODULE_API_FUNC(RedisModule_Log)(RedisModuleCtx *ctx, const char *level, const char *fmt, ...); __attribute__ ((format (printf, 3, 4)));
-void REDISMODULE_API_FUNC(RedisModule_LogIOError)(RedisModuleIO *io, const char *levelstr, const char *fmt, ...); __attribute__ ((format (printf, 3, 4)));
+#ifdef __GNUC__
+void REDISMODULE_API_FUNC(RedisModule_Log)(RedisModuleCtx *ctx, const char *level, const char *fmt, ...) __attribute__ ((format (printf, 3, 4)));
+void REDISMODULE_API_FUNC(RedisModule_LogIOError)(RedisModuleIO *io, const char *levelstr, const char *fmt, ...) __attribute__ ((format (printf, 3, 4)));
+#else
+void REDISMODULE_API_FUNC(RedisModule_Log)(RedisModuleCtx *ctx, const char *level, const char *fmt, ...);
+void REDISMODULE_API_FUNC(RedisModule_LogIOError)(RedisModuleIO *io, const char *levelstr, const char *fmt, ...);
+#endif
 void REDISMODULE_API_FUNC(RedisModule__Assert)(const char *estr, const char *file, int line);
 void REDISMODULE_API_FUNC(RedisModule_LatencyAddSample)(const char *event, mstime_t latency);
 int REDISMODULE_API_FUNC(RedisModule_StringAppendBuffer)(RedisModuleCtx *ctx, RedisModuleString *str, const char *buf, size_t len);

From f695d183068fa8f1b6315c08c0d72515e85aea55 Mon Sep 17 00:00:00 2001
From: Guy Benoish <guy.benoish@redislabs.com>
Date: Thu, 2 Apr 2020 14:57:17 +0300
Subject: [PATCH 021/186] Try to fix time-sensitive tests in blockonkey.tcl

There is an inherent race between the deferring client and the
"main" client of the test: While the deferring client issues a blocking
command, we can't know for sure that by the time the "main" client
tries to issue another command (Usually one that unblocks the deferring
client) the deferring client is even blocked...
For lack of a better choice this commit uses TCL's 'after' in order
to give some time for the deferring client to issues its blocking
command before the "main" client does its thing.
This problem probably exists in many other tests but this commit
tries to fix blockonkeys.tcl
---
 tests/unit/moduleapi/blockonkeys.tcl | 55 +++++++++++++++++++++++++++-
 1 file changed, 54 insertions(+), 1 deletion(-)

diff --git a/tests/unit/moduleapi/blockonkeys.tcl b/tests/unit/moduleapi/blockonkeys.tcl
index c8b8f23ed..5e5d93da3 100644
--- a/tests/unit/moduleapi/blockonkeys.tcl
+++ b/tests/unit/moduleapi/blockonkeys.tcl
@@ -11,6 +11,12 @@ start_server {tags {"modules"}} {
 
         $rd1 fsl.bpoppush src dst 0
         $rd2 fsl.bpoppush dst src 0
+        ;# wait until clients are actually blocked
+        wait_for_condition 50 100 {
+            [s 0 blocked_clients] eq {2}
+        } else {
+            fail "Clients are not blocked"
+        }
 
         r fsl.push src 42
 
@@ -24,7 +30,12 @@ start_server {tags {"modules"}} {
         r del src
 
         $rd1 fsl.bpoppush src src 0
-
+        ;# wait until clients are actually blocked
+        wait_for_condition 50 100 {
+            [s 0 blocked_clients] eq {1}
+        } else {
+            fail "Clients are not blocked"
+        }
         r fsl.push src 42
 
         assert_equal {42} [r fsl.getall src]
@@ -48,6 +59,12 @@ start_server {tags {"modules"}} {
         r del k
         set rd [redis_deferring_client]
         $rd fsl.bpop k 0
+        ;# wait until clients are actually blocked
+        wait_for_condition 50 100 {
+            [s 0 blocked_clients] eq {1}
+        } else {
+            fail "Clients are not blocked"
+        }
         r fsl.push k 34
         assert_equal {34} [$rd read]
     }
@@ -76,6 +93,12 @@ start_server {tags {"modules"}} {
         set cid [$rd read]
         r fsl.push k 33
         $rd fsl.bpopgt k 33 0
+        ;# wait until clients are actually blocked
+        wait_for_condition 50 100 {
+            [s 0 blocked_clients] eq {1}
+        } else {
+            fail "Clients are not blocked"
+        }
         r fsl.push k 34
         assert_equal {34} [$rd read]
         r client kill id $cid ;# try to smoke-out client-related memory leak
@@ -85,6 +108,12 @@ start_server {tags {"modules"}} {
         r del k
         set rd [redis_deferring_client]
         $rd fsl.bpopgt k 35 0
+        ;# wait until clients are actually blocked
+        wait_for_condition 50 100 {
+            [s 0 blocked_clients] eq {1}
+        } else {
+            fail "Clients are not blocked"
+        }
         r fsl.push k 33
         r fsl.push k 34
         r fsl.push k 35
@@ -98,6 +127,12 @@ start_server {tags {"modules"}} {
         $rd client id
         set cid [$rd read]
         $rd fsl.bpopgt k 35 0
+        ;# wait until clients are actually blocked
+        wait_for_condition 50 100 {
+            [s 0 blocked_clients] eq {1}
+        } else {
+            fail "Clients are not blocked"
+        }
         r client kill id $cid ;# try to smoke-out client-related memory leak
     }
 
@@ -107,6 +142,12 @@ start_server {tags {"modules"}} {
         $rd client id
         set cid [$rd read]
         $rd fsl.bpopgt k 35 0
+        ;# wait until clients are actually blocked
+        wait_for_condition 50 100 {
+            [s 0 blocked_clients] eq {1}
+        } else {
+            fail "Clients are not blocked"
+        }
         r client unblock $cid timeout ;# try to smoke-out client-related memory leak
         assert_equal {Request timedout} [$rd read]
     }
@@ -117,6 +158,12 @@ start_server {tags {"modules"}} {
         $rd client id
         set cid [$rd read]
         $rd fsl.bpopgt k 35 0
+        ;# wait until clients are actually blocked
+        wait_for_condition 50 100 {
+            [s 0 blocked_clients] eq {1}
+        } else {
+            fail "Clients are not blocked"
+        }
         r client unblock $cid error ;# try to smoke-out client-related memory leak
         assert_error "*unblocked*" {$rd read}
     }
@@ -125,6 +172,12 @@ start_server {tags {"modules"}} {
         r del k
         set rd [redis_deferring_client]
         $rd fsl.bpop k 0
+        ;# wait until clients are actually blocked
+        wait_for_condition 50 100 {
+            [s 0 blocked_clients] eq {1}
+        } else {
+            fail "Clients are not blocked"
+        }
         r lpush k 12
         r lpush k 13
         r lpush k 14

From 02b594f6adaf58db5497e766b46e742ed48fb980 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Mon, 6 Apr 2020 09:41:14 +0300
Subject: [PATCH 022/186] diffrent fix for runtest --host --port

---
 tests/support/server.tcl |  7 ++-----
 tests/test_helper.tcl    | 19 +++++++++++--------
 2 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/tests/support/server.tcl b/tests/support/server.tcl
index 400017c5f..d086366dc 100644
--- a/tests/support/server.tcl
+++ b/tests/support/server.tcl
@@ -159,12 +159,9 @@ proc start_server {options {code undefined}} {
     if {$::external} {
         if {[llength $::servers] == 0} {
             set srv {}
-            # In test_server_main(tests/test_helper.tcl:215~218), increase the value of start_port
-            # and assign it to ::port through the `--port` option, so we need to reduce it.
-            set baseport [expr {$::port-100}]
             dict set srv "host" $::host
-            dict set srv "port" $baseport
-            set client [redis $::host $baseport 0 $::tls]
+            dict set srv "port" $::port
+            set client [redis $::host $::port 0 $::tls]
             dict set srv "client" $client
             $client select 9
 
diff --git a/tests/test_helper.tcl b/tests/test_helper.tcl
index 5cb43104b..d80cb6907 100644
--- a/tests/test_helper.tcl
+++ b/tests/test_helper.tcl
@@ -212,13 +212,19 @@ proc test_server_main {} {
 
     # Start the client instances
     set ::clients_pids {}
-    set start_port [expr {$::port+100}]
-    for {set j 0} {$j < $::numclients} {incr j} {
-        set start_port [find_available_port $start_port]
+    if {$::external} {
         set p [exec $tclsh [info script] {*}$::argv \
-            --client $port --port $start_port &]
+            --client $port --port $::port &]
         lappend ::clients_pids $p
-        incr start_port 10
+    } else {
+        set start_port [expr {$::port+100}]
+        for {set j 0} {$j < $::numclients} {incr j} {
+            set start_port [find_available_port $start_port]
+            set p [exec $tclsh [info script] {*}$::argv \
+                --client $port --port $start_port &]
+            lappend ::clients_pids $p
+            incr start_port 10
+        }
     }
 
     # Setup global state for the test server
@@ -506,9 +512,6 @@ for {set j 0} {$j < [llength $argv]} {incr j} {
     } elseif {$opt eq {--host}} {
         set ::external 1
         set ::host $arg
-        # If we use an external server, we can only set numclients to 1,
-        # otherwise the port will be miscalculated.
-        set ::numclients 1
         incr j
     } elseif {$opt eq {--port}} {
         set ::port $arg

From 026cc11b056f063631d990f1a9db45b4e583974e Mon Sep 17 00:00:00 2001
From: srzhao <srzhao@sysnew.com>
Date: Mon, 20 Jan 2020 21:17:02 +0800
Subject: [PATCH 023/186] Check OOM at script start to get stable lua OOM
 state.

Checking OOM by `getMaxMemoryState` inside script might get different result
with `freeMemoryIfNeededAndSafe` at script start, because lua stack and
arguments also consume memory.

This leads to memory `borderline` when memory grows near server.maxmemory:

- `freeMemoryIfNeededAndSafe` at script start detects no OOM, no memory freed
- `getMaxMemoryState` inside script detects OOM, script aborted

We solve this 'borderline' issue by saving OOM state at script start to get
stable lua OOM state.

related to issue #6565 and #5250.
---
 src/scripting.c | 7 +++----
 src/server.c    | 7 +++++++
 src/server.h    | 1 +
 3 files changed, 11 insertions(+), 4 deletions(-)

diff --git a/src/scripting.c b/src/scripting.c
index 7f64e06db..32a511e13 100644
--- a/src/scripting.c
+++ b/src/scripting.c
@@ -657,12 +657,11 @@ int luaRedisGenericCommand(lua_State *lua, int raise_error) {
         !server.loading &&              /* Don't care about mem if loading. */
         !server.masterhost &&           /* Slave must execute the script. */
         server.lua_write_dirty == 0 &&  /* Script had no side effects so far. */
+        server.lua_oom &&               /* Detected OOM when script start. */
         (cmd->flags & CMD_DENYOOM))
     {
-        if (getMaxmemoryState(NULL,NULL,NULL,NULL) != C_OK) {
-            luaPushError(lua, shared.oomerr->ptr);
-            goto cleanup;
-        }
+        luaPushError(lua, shared.oomerr->ptr);
+        goto cleanup;
     }
 
     if (cmd->flags & CMD_RANDOM) server.lua_random_dirty = 1;
diff --git a/src/server.c b/src/server.c
index 9ebf0ee6b..d6abc72d9 100644
--- a/src/server.c
+++ b/src/server.c
@@ -3441,6 +3441,13 @@ int processCommand(client *c) {
             addReply(c, shared.oomerr);
             return C_OK;
         }
+
+        /* Save out_of_memory result at script start, otherwise if we check OOM
+         * untill first write within script, memory used by lua stack and
+         * arguments might interfere. */
+        if (c->cmd->proc == evalCommand || c->cmd->proc == evalShaCommand) {
+            server.lua_oom = out_of_memory;
+        }
     }
 
     /* Make sure to use a reasonable amount of memory for client side
diff --git a/src/server.h b/src/server.h
index dbd72281f..f37fe4b12 100644
--- a/src/server.h
+++ b/src/server.h
@@ -1392,6 +1392,7 @@ struct redisServer {
                              execution. */
     int lua_kill;         /* Kill the script if true. */
     int lua_always_replicate_commands; /* Default replication type. */
+    int lua_oom;          /* OOM detected when script start? */
     /* Lazy free */
     int lazyfree_lazy_eviction;
     int lazyfree_lazy_expire;

From a518a9a76660849a89082fa09750561ad60ad468 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Wed, 1 Apr 2020 16:10:18 +0200
Subject: [PATCH 024/186] LCS: initial functionality implemented.

---
 src/db.c       |  26 +++++++++++
 src/server.c   |   6 ++-
 src/server.h   |   2 +
 src/t_string.c | 123 +++++++++++++++++++++++++++++++++++++++++++++++++
 4 files changed, 156 insertions(+), 1 deletion(-)

diff --git a/src/db.c b/src/db.c
index cadfeb77b..08247fa3c 100644
--- a/src/db.c
+++ b/src/db.c
@@ -1554,6 +1554,32 @@ int *georadiusGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numk
     return keys;
 }
 
+/* LCS ... [STOREIDX <key>] ... */
+int *lcsGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numkeys)
+{
+    int i;
+    int *keys;
+    UNUSED(cmd);
+
+    /* We need to parse the options of the command in order to check for the
+     * "STOREIDX" argument before the STRINGS argument. */
+    for (i = 1; i < argc; i++) {
+        char *arg = argv[i]->ptr;
+        int moreargs = (argc-1) - i;
+
+        if (!strcasecmp(arg, "strings")) {
+            break;
+        } else if (!strcasecmp(arg, "storeidx") && moreargs) {
+            keys = getKeysTempBuffer;
+            keys[0] = i+1;
+            *numkeys = 1;
+            return keys;
+        }
+    }
+    *numkeys = 0;
+    return NULL;
+}
+
 /* Helper function to extract keys from memory command.
  * MEMORY USAGE <key> */
 int *memoryGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numkeys) {
diff --git a/src/server.c b/src/server.c
index d6abc72d9..56feb09a3 100644
--- a/src/server.c
+++ b/src/server.c
@@ -1004,7 +1004,11 @@ struct redisCommand redisCommandTable[] = {
 
     {"acl",aclCommand,-2,
      "admin no-script no-slowlog ok-loading ok-stale",
-     0,NULL,0,0,0,0,0,0}
+     0,NULL,0,0,0,0,0,0},
+
+    {"lcs",lcsCommand,-4,
+     "write use-memory @string",
+     0,lcsGetKeys,0,0,0,0,0,0}
 };
 
 /*============================ Utility functions ============================ */
diff --git a/src/server.h b/src/server.h
index f37fe4b12..b17995948 100644
--- a/src/server.h
+++ b/src/server.h
@@ -2102,6 +2102,7 @@ int *migrateGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numkey
 int *georadiusGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numkeys);
 int *xreadGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numkeys);
 int *memoryGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numkeys);
+int *lcsGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numkeys);
 
 /* Cluster */
 void clusterInit(void);
@@ -2373,6 +2374,7 @@ void xdelCommand(client *c);
 void xtrimCommand(client *c);
 void lolwutCommand(client *c);
 void aclCommand(client *c);
+void lcsCommand(client *c);
 
 #if defined(__GNUC__)
 void *calloc(size_t count, size_t size) __attribute__ ((deprecated));
diff --git a/src/t_string.c b/src/t_string.c
index 3174e9ccd..246b06beb 100644
--- a/src/t_string.c
+++ b/src/t_string.c
@@ -479,3 +479,126 @@ void strlenCommand(client *c) {
         checkType(c,o,OBJ_STRING)) return;
     addReplyLongLong(c,stringObjectLen(o));
 }
+
+/* LCS -- Longest common subsequence.
+ *
+ * LCS [LEN] [IDX] [STOREIDX <key>] STRINGS <string> <string> */
+void lcsCommand(client *c) {
+    uint32_t i, j;
+    sds a = NULL, b = NULL;
+    int getlen = 0, getidx = 0;
+    robj *idxkey = NULL; /* STOREIDX will set this and getidx to 1. */
+
+    for (j = 1; j < (uint32_t)c->argc; j++) {
+        char *opt = c->argv[j]->ptr;
+        int moreargs = (c->argc-1) - j;
+
+        if (!strcasecmp(opt,"IDX")) {
+            getidx = 1;
+        } else if (!strcasecmp(opt,"STOREIDX") && moreargs) {
+            getidx = 1;
+            idxkey = c->argv[j+1];
+            j++;
+        } else if (!strcasecmp(opt,"LEN")) {
+            getlen++;
+        } else if (!strcasecmp(opt,"STRINGS")) {
+            if (moreargs != 2) {
+                addReplyError(c,"LCS requires exactly two strings");
+                return;
+            }
+            a = c->argv[j+1]->ptr;
+            b = c->argv[j+2]->ptr;
+            j += 2;
+        } else {
+            addReply(c,shared.syntaxerr);
+            return;
+        }
+    }
+
+    /* Complain if the user didn't pass the STRING option. */
+    if (a == NULL) {
+        addReplyError(c,"STRINGS <string-a> <string-b> is mandatory");
+        return;
+    }
+
+    /* Compute the LCS using the vanilla dynamic programming technique of
+     * building a table of LCS(x,y) substrings. */
+    uint32_t alen = sdslen(a);
+    uint32_t blen = sdslen(b);
+
+    /* Setup an uint32_t array to store at LCS[i,j] the length of the
+     * LCS A0..i-1, B0..j-1. Note that we have a linear array here, so
+     * we index it as LCS[i+alen*j] */
+    uint32_t *lcs = zmalloc((alen+1)*(blen+1)*sizeof(uint32_t));
+    #define LCS(A,B) lcs[(A)+((B)*(alen+1))]
+
+    /* Start building the LCS table. */
+    for (uint32_t i = 0; i <= alen; i++) {
+        for (uint32_t j = 0; j <= blen; j++) {
+            if (i == 0 || j == 0) {
+                /* If one substring has length of zero, the
+                 * LCS length is zero. */
+                LCS(i,j) = 0;
+            } else if (a[i-1] == b[j-1]) {
+                /* The len LCS (and the LCS itself) of two
+                 * sequences with the same final character, is the
+                 * LCS of the two sequences without the last char
+                 * plus that last char. */
+                LCS(i,j) = LCS(i-1,j-1)+1;
+            } else {
+                /* If the last character is different, take the longest
+                 * between the LCS of the first string and the second
+                 * minus the last char, and the reverse. */
+                uint32_t lcs1 = LCS(i-1,j);
+                uint32_t lcs2 = LCS(i,j-1);
+                LCS(i,j) = lcs1 > lcs2 ? lcs1 : lcs2;
+            }
+        }
+    }
+
+    /* Store the actual LCS string in "result" if needed. We create
+     * it backward, but the length is already known, we store it into idx. */
+    uint32_t idx = LCS(alen,blen);
+    sds result = NULL;
+
+    /* Do we need to compute the actual LCS string? Allocate it in that case. */
+    int computelcs = getidx || !getlen;
+    if (computelcs) result = sdsnewlen(SDS_NOINIT,idx);
+
+    i = alen, j = blen;
+    while (computelcs && i > 0 && j > 0) {
+        if (a[i-1] == b[j-1]) {
+            /* If there is a match, store the character and reduce
+             * the indexes to look for a new match. */
+            result[idx-1] = a[i-1];
+            idx--;
+            i--;
+            j--;
+        } else {
+            /* Otherwise reduce i and j depending on the largest
+             * LCS between, to understand what direction we need to go. */
+            uint32_t lcs1 = LCS(i-1,j);
+            uint32_t lcs2 = LCS(i,j-1);
+            if (lcs1 > lcs2)
+                i--;
+            else
+                j--;
+        }
+    }
+
+    /* Signal modified key, increment dirty, ... */
+
+    /* Reply depending on the given options. */
+    if (getlen) {
+        addReplyLongLong(c,LCS(alen,blen));
+    } else {
+        addReplyBulkSds(c,result);
+        result = NULL;
+    }
+
+    /* Cleanup. */
+    sdsfree(result);
+    zfree(lcs);
+    return;
+}
+

From 420aac727447a10d11226222655c89fbab6f6239 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Wed, 1 Apr 2020 17:11:31 +0200
Subject: [PATCH 025/186] LCS: implement range indexes option.

---
 src/t_string.c | 68 +++++++++++++++++++++++++++++++++++++++++++-------
 1 file changed, 59 insertions(+), 9 deletions(-)

diff --git a/src/t_string.c b/src/t_string.c
index 246b06beb..a1481c236 100644
--- a/src/t_string.c
+++ b/src/t_string.c
@@ -482,7 +482,7 @@ void strlenCommand(client *c) {
 
 /* LCS -- Longest common subsequence.
  *
- * LCS [LEN] [IDX] [STOREIDX <key>] STRINGS <string> <string> */
+ * LCS [IDX] [STOREIDX <key>] STRINGS <string> <string> */
 void lcsCommand(client *c) {
     uint32_t i, j;
     sds a = NULL, b = NULL;
@@ -495,12 +495,12 @@ void lcsCommand(client *c) {
 
         if (!strcasecmp(opt,"IDX")) {
             getidx = 1;
+        } else if (!strcasecmp(opt,"LEN")) {
+            getlen = 1;
         } else if (!strcasecmp(opt,"STOREIDX") && moreargs) {
             getidx = 1;
             idxkey = c->argv[j+1];
             j++;
-        } else if (!strcasecmp(opt,"LEN")) {
-            getlen++;
         } else if (!strcasecmp(opt,"STRINGS")) {
             if (moreargs != 2) {
                 addReplyError(c,"LCS requires exactly two strings");
@@ -515,10 +515,17 @@ void lcsCommand(client *c) {
         }
     }
 
-    /* Complain if the user didn't pass the STRING option. */
+    /* Complain if the user passed ambiguous parameters. */
     if (a == NULL) {
         addReplyError(c,"STRINGS <string-a> <string-b> is mandatory");
         return;
+    } else if (getlen && (getidx && idxkey == NULL)) {
+        addReplyError(c,
+            "If you want both the LEN and indexes, please "
+            "store the indexes into a key with STOREIDX. However note "
+            "that the IDX output also includes both the LCS string and "
+            "its length");
+        return;
     }
 
     /* Compute the LCS using the vanilla dynamic programming technique of
@@ -559,21 +566,62 @@ void lcsCommand(client *c) {
     /* Store the actual LCS string in "result" if needed. We create
      * it backward, but the length is already known, we store it into idx. */
     uint32_t idx = LCS(alen,blen);
-    sds result = NULL;
+    sds result = NULL;        /* Resulting LCS string. */
+    void *arraylenptr = NULL; /* Deffered length of the array for IDX. */
+    uint32_t arange_start = alen, /* alen signals that values are not set. */
+             arange_end = 0,
+             brange_start = 0,
+             brange_end = 0;
 
     /* Do we need to compute the actual LCS string? Allocate it in that case. */
     int computelcs = getidx || !getlen;
     if (computelcs) result = sdsnewlen(SDS_NOINIT,idx);
 
+    /* Start with a deferred array if we have to emit the ranges. */
+    uint32_t arraylen = 0;  /* Number of ranges emitted in the array. */
+    if (getidx && idxkey == NULL)
+        arraylenptr = addReplyDeferredLen(c);
+
     i = alen, j = blen;
     while (computelcs && i > 0 && j > 0) {
         if (a[i-1] == b[j-1]) {
             /* If there is a match, store the character and reduce
              * the indexes to look for a new match. */
             result[idx-1] = a[i-1];
-            idx--;
-            i--;
-            j--;
+            /* Track the current range. */
+            int emit_range = 0;
+            if (arange_start == alen) {
+                arange_start = i-1;
+                arange_end = i-1;
+                brange_start = j-1;
+                brange_end = j-1;
+                if (i == 0 || j == 0) emit_range = 1;
+            } else {
+                /* Let's see if we can extend the range backward since
+                 * it is contiguous. */
+                if (arange_start == i && brange_start == j) {
+                    arange_start--;
+                    brange_start--;
+                } else {
+                    emit_range = 1;
+                }
+            }
+
+            /* Emit the current range if needed. */
+            if (emit_range) {
+                if (arraylenptr) {
+                    addReplyArrayLen(c,2);
+                    addReplyArrayLen(c,2);
+                    addReplyLongLong(c,arange_start);
+                    addReplyLongLong(c,arange_end);
+                    addReplyArrayLen(c,2);
+                    addReplyLongLong(c,brange_start);
+                    addReplyLongLong(c,brange_end);
+                }
+                arange_start = alen; /* Restart at the next match. */
+                arraylen++;
+            }
+            idx--; i--; j--;
         } else {
             /* Otherwise reduce i and j depending on the largest
              * LCS between, to understand what direction we need to go. */
@@ -589,7 +637,9 @@ void lcsCommand(client *c) {
     /* Signal modified key, increment dirty, ... */
 
     /* Reply depending on the given options. */
-    if (getlen) {
+    if (arraylenptr) {
+        setDeferredArrayLen(c,arraylenptr,arraylen);
+    } else if (getlen) {
         addReplyLongLong(c,LCS(alen,blen));
     } else {
         addReplyBulkSds(c,result);

From 2b67b6b87e7aff0a89c4c4fefd4b4ef3305dcac6 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Wed, 1 Apr 2020 17:14:36 +0200
Subject: [PATCH 026/186] LCS: fix emission of last range starting at index 0.

---
 src/t_string.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/t_string.c b/src/t_string.c
index a1481c236..e7d302066 100644
--- a/src/t_string.c
+++ b/src/t_string.c
@@ -595,7 +595,6 @@ void lcsCommand(client *c) {
                 arange_end = i-1;
                 brange_start = j-1;
                 brange_end = j-1;
-                if (i == 0 || j == 0) emit_range = 1;
             } else {
                 /* Let's see if we can extend the range backward since
                  * it is contiguous. */
@@ -606,6 +605,7 @@ void lcsCommand(client *c) {
                     emit_range = 1;
                 }
             }
+            if (arange_start == 0 || brange_start == 0) emit_range = 1;
 
             /* Emit the current range if needed. */
             if (emit_range) {

From 4aa24e62a4fd9336875e96f8a0d75fcc049e5d28 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Wed, 1 Apr 2020 17:36:32 +0200
Subject: [PATCH 027/186] LCS: other fixes to range emission.

---
 src/t_string.c | 36 ++++++++++++++++++++----------------
 1 file changed, 20 insertions(+), 16 deletions(-)

diff --git a/src/t_string.c b/src/t_string.c
index e7d302066..e44971758 100644
--- a/src/t_string.c
+++ b/src/t_string.c
@@ -584,12 +584,13 @@ void lcsCommand(client *c) {
 
     i = alen, j = blen;
     while (computelcs && i > 0 && j > 0) {
+        int emit_range = 0;
         if (a[i-1] == b[j-1]) {
             /* If there is a match, store the character and reduce
              * the indexes to look for a new match. */
             result[idx-1] = a[i-1];
+
             /* Track the current range. */
-            int emit_range = 0;
             if (arange_start == alen) {
                 arange_start = i-1;
                 arange_end = i-1;
@@ -605,22 +606,9 @@ void lcsCommand(client *c) {
                     emit_range = 1;
                 }
             }
+            /* Emit the range if we matched with the first byte of
+             * one of the two strings. We'll exit the loop ASAP. */
             if (arange_start == 0 || brange_start == 0) emit_range = 1;
-
-            /* Emit the current range if needed. */
-            if (emit_range) {
-                if (arraylenptr) {
-                    addReplyArrayLen(c,2);
-                    addReplyArrayLen(c,2);
-                    addReplyLongLong(c,arange_start);
-                    addReplyLongLong(c,arange_end);
-                    addReplyArrayLen(c,2);
-                    addReplyLongLong(c,brange_start);
-                    addReplyLongLong(c,brange_end);
-                }
-                arange_start = alen; /* Restart at the next match. */
-                arraylen++;
-            }
             idx--; i--; j--;
         } else {
             /* Otherwise reduce i and j depending on the largest
@@ -631,6 +619,22 @@ void lcsCommand(client *c) {
                 i--;
             else
                 j--;
+            if (arange_start != alen) emit_range = 1;
+        }
+
+        /* Emit the current range if needed. */
+        if (emit_range) {
+            if (arraylenptr) {
+                addReplyArrayLen(c,2);
+                addReplyArrayLen(c,2);
+                addReplyLongLong(c,arange_start);
+                addReplyLongLong(c,arange_end);
+                addReplyArrayLen(c,2);
+                addReplyLongLong(c,brange_start);
+                addReplyLongLong(c,brange_end);
+            }
+            arange_start = alen; /* Restart at the next match. */
+            arraylen++;
         }
     }
 

From a9f8a8cba0b7d746e063c33a7a07d665df654faa Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Wed, 1 Apr 2020 22:11:59 +0200
Subject: [PATCH 028/186] LCS: implement KEYS option.

---
 src/t_string.c | 20 ++++++++++++++++++--
 1 file changed, 18 insertions(+), 2 deletions(-)

diff --git a/src/t_string.c b/src/t_string.c
index e44971758..fbcb1a23a 100644
--- a/src/t_string.c
+++ b/src/t_string.c
@@ -482,12 +482,13 @@ void strlenCommand(client *c) {
 
 /* LCS -- Longest common subsequence.
  *
- * LCS [IDX] [STOREIDX <key>] STRINGS <string> <string> */
+ * LCS [IDX] [STOREIDX <key>] STRINGS <string> <string> | KEYS <keya> <keyb> */
 void lcsCommand(client *c) {
     uint32_t i, j;
     sds a = NULL, b = NULL;
     int getlen = 0, getidx = 0;
     robj *idxkey = NULL; /* STOREIDX will set this and getidx to 1. */
+    robj *obja = NULL, *objb = NULL;
 
     for (j = 1; j < (uint32_t)c->argc; j++) {
         char *opt = c->argv[j]->ptr;
@@ -509,6 +510,18 @@ void lcsCommand(client *c) {
             a = c->argv[j+1]->ptr;
             b = c->argv[j+2]->ptr;
             j += 2;
+        } else if (!strcasecmp(opt,"KEYS")) {
+            if (moreargs != 2) {
+                addReplyError(c,"LCS requires exactly two keys");
+                return;
+            }
+            obja = lookupKeyRead(c->db,c->argv[j+1]);
+            objb = lookupKeyRead(c->db,c->argv[j+2]);
+            obja = obja ? getDecodedObject(obja) : createStringObject("",0);
+            objb = objb ? getDecodedObject(objb) : createStringObject("",0);
+            a = obja->ptr;
+            b = objb->ptr;
+            j += 2;
         } else {
             addReply(c,shared.syntaxerr);
             return;
@@ -517,7 +530,8 @@ void lcsCommand(client *c) {
 
     /* Complain if the user passed ambiguous parameters. */
     if (a == NULL) {
-        addReplyError(c,"STRINGS <string-a> <string-b> is mandatory");
+        addReplyError(c,"Please specify two strings: "
+                        "STRINGS or KEYS options are mandatory");
         return;
     } else if (getlen && (getidx && idxkey == NULL)) {
         addReplyError(c,
@@ -651,6 +665,8 @@ void lcsCommand(client *c) {
     }
 
     /* Cleanup. */
+    if (obja) decrRefCount(obja);
+    if (objb) decrRefCount(objb);
     sdsfree(result);
     zfree(lcs);
     return;

From ebb09a5c3b3170b9f54c6b951f1d9ff53e60b967 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Wed, 1 Apr 2020 23:45:07 +0200
Subject: [PATCH 029/186] LCS: 7x speedup by accessing the array with better
 locality.

---
 src/t_string.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/t_string.c b/src/t_string.c
index fbcb1a23a..91a2017e3 100644
--- a/src/t_string.c
+++ b/src/t_string.c
@@ -551,7 +551,7 @@ void lcsCommand(client *c) {
      * LCS A0..i-1, B0..j-1. Note that we have a linear array here, so
      * we index it as LCS[i+alen*j] */
     uint32_t *lcs = zmalloc((alen+1)*(blen+1)*sizeof(uint32_t));
-    #define LCS(A,B) lcs[(A)+((B)*(alen+1))]
+    #define LCS(A,B) lcs[(B)+((A)*(blen+1))]
 
     /* Start building the LCS table. */
     for (uint32_t i = 0; i <= alen; i++) {

From 56a52e804c75979ff77a97e3f65e8976ccb2a436 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Thu, 2 Apr 2020 13:37:35 +0200
Subject: [PATCH 030/186] LCS: MINMATCHLEN and WITHMATCHLEN options.

---
 src/t_string.c | 35 ++++++++++++++++++++++++-----------
 1 file changed, 24 insertions(+), 11 deletions(-)

diff --git a/src/t_string.c b/src/t_string.c
index 91a2017e3..70421f703 100644
--- a/src/t_string.c
+++ b/src/t_string.c
@@ -482,11 +482,13 @@ void strlenCommand(client *c) {
 
 /* LCS -- Longest common subsequence.
  *
- * LCS [IDX] [STOREIDX <key>] STRINGS <string> <string> | KEYS <keya> <keyb> */
+ * LCS [IDX] [STOREIDX <key>] [MINMATCHLEN <len>]
+ *     STRINGS <string> <string> | KEYS <keya> <keyb> */
 void lcsCommand(client *c) {
     uint32_t i, j;
+    long long minmatchlen = 0;
     sds a = NULL, b = NULL;
-    int getlen = 0, getidx = 0;
+    int getlen = 0, getidx = 0, withmatchlen = 0;
     robj *idxkey = NULL; /* STOREIDX will set this and getidx to 1. */
     robj *obja = NULL, *objb = NULL;
 
@@ -498,10 +500,17 @@ void lcsCommand(client *c) {
             getidx = 1;
         } else if (!strcasecmp(opt,"LEN")) {
             getlen = 1;
+        } else if (!strcasecmp(opt,"WITHMATCHLEN")) {
+            withmatchlen = 1;
         } else if (!strcasecmp(opt,"STOREIDX") && moreargs) {
             getidx = 1;
             idxkey = c->argv[j+1];
             j++;
+        } else if (!strcasecmp(opt,"MINMATCHLEN") && moreargs) {
+            if (getLongLongFromObjectOrReply(c,c->argv[j+1],&minmatchlen,NULL)
+                != C_OK) return;
+            if (minmatchlen < 0) minmatchlen = 0;
+            j++;
         } else if (!strcasecmp(opt,"STRINGS")) {
             if (moreargs != 2) {
                 addReplyError(c,"LCS requires exactly two strings");
@@ -637,18 +646,22 @@ void lcsCommand(client *c) {
         }
 
         /* Emit the current range if needed. */
+        uint32_t match_len = arange_end - arange_start + 1;
         if (emit_range) {
-            if (arraylenptr) {
-                addReplyArrayLen(c,2);
-                addReplyArrayLen(c,2);
-                addReplyLongLong(c,arange_start);
-                addReplyLongLong(c,arange_end);
-                addReplyArrayLen(c,2);
-                addReplyLongLong(c,brange_start);
-                addReplyLongLong(c,brange_end);
+            if (minmatchlen == 0 || match_len >= minmatchlen) {
+                if (arraylenptr) {
+                    addReplyArrayLen(c,2+withmatchlen);
+                    addReplyArrayLen(c,2);
+                    addReplyLongLong(c,arange_start);
+                    addReplyLongLong(c,arange_end);
+                    addReplyArrayLen(c,2);
+                    addReplyLongLong(c,brange_start);
+                    addReplyLongLong(c,brange_end);
+                    if (withmatchlen) addReplyLongLong(c,match_len);
+                    arraylen++;
+                }
             }
             arange_start = alen; /* Restart at the next match. */
-            arraylen++;
         }
     }
 

From cb92c23de052f88f9517e317eab431cadaa8e0f6 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Thu, 2 Apr 2020 16:15:17 +0200
Subject: [PATCH 031/186] LCS: output LCS len as well in IDX mode.

---
 src/t_string.c | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/src/t_string.c b/src/t_string.c
index 70421f703..b884aee98 100644
--- a/src/t_string.c
+++ b/src/t_string.c
@@ -602,8 +602,11 @@ void lcsCommand(client *c) {
 
     /* Start with a deferred array if we have to emit the ranges. */
     uint32_t arraylen = 0;  /* Number of ranges emitted in the array. */
-    if (getidx && idxkey == NULL)
+    if (getidx && idxkey == NULL) {
+        addReplyMapLen(c,2);
+        addReplyBulkCString(c,"matches");
         arraylenptr = addReplyDeferredLen(c);
+    }
 
     i = alen, j = blen;
     while (computelcs && i > 0 && j > 0) {
@@ -669,6 +672,8 @@ void lcsCommand(client *c) {
 
     /* Reply depending on the given options. */
     if (arraylenptr) {
+        addReplyBulkCString(c,"len");
+        addReplyLongLong(c,LCS(alen,blen));
         setDeferredArrayLen(c,arraylenptr,arraylen);
     } else if (getlen) {
         addReplyLongLong(c,LCS(alen,blen));

From a4c49070355c86206d6ad3c556e363f860b8140d Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Thu, 2 Apr 2020 21:17:31 +0200
Subject: [PATCH 032/186] LCS: fix stale comment.

---
 src/t_string.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/t_string.c b/src/t_string.c
index b884aee98..045d73ca6 100644
--- a/src/t_string.c
+++ b/src/t_string.c
@@ -558,7 +558,7 @@ void lcsCommand(client *c) {
 
     /* Setup an uint32_t array to store at LCS[i,j] the length of the
      * LCS A0..i-1, B0..j-1. Note that we have a linear array here, so
-     * we index it as LCS[i+alen*j] */
+     * we index it as LCS[j+(blen+1)*j] */
     uint32_t *lcs = zmalloc((alen+1)*(blen+1)*sizeof(uint32_t));
     #define LCS(A,B) lcs[(B)+((A)*(blen+1))]
 

From 9254a805d7ce9d946cbaf179f977abe677969997 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Mon, 6 Apr 2020 13:23:07 +0200
Subject: [PATCH 033/186] LCS: get rid of STOREIDX option. Fix get keys helper.

---
 src/db.c       | 14 +++++++-------
 src/t_string.c | 27 +++++++++++++--------------
 2 files changed, 20 insertions(+), 21 deletions(-)

diff --git a/src/db.c b/src/db.c
index 08247fa3c..6e5a8bf3a 100644
--- a/src/db.c
+++ b/src/db.c
@@ -1554,30 +1554,30 @@ int *georadiusGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numk
     return keys;
 }
 
-/* LCS ... [STOREIDX <key>] ... */
+/* LCS ... [KEYS <key1> <key2>] ... */
 int *lcsGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numkeys)
 {
     int i;
-    int *keys;
+    int *keys = getKeysTempBuffer;
     UNUSED(cmd);
 
     /* We need to parse the options of the command in order to check for the
-     * "STOREIDX" argument before the STRINGS argument. */
+     * "KEYS" argument before the "STRINGS" argument. */
     for (i = 1; i < argc; i++) {
         char *arg = argv[i]->ptr;
         int moreargs = (argc-1) - i;
 
         if (!strcasecmp(arg, "strings")) {
             break;
-        } else if (!strcasecmp(arg, "storeidx") && moreargs) {
-            keys = getKeysTempBuffer;
+        } else if (!strcasecmp(arg, "keys") && moreargs >= 2) {
             keys[0] = i+1;
-            *numkeys = 1;
+            keys[1] = i+2;
+            *numkeys = 2;
             return keys;
         }
     }
     *numkeys = 0;
-    return NULL;
+    return keys;
 }
 
 /* Helper function to extract keys from memory command.
diff --git a/src/t_string.c b/src/t_string.c
index 045d73ca6..39eb055fb 100644
--- a/src/t_string.c
+++ b/src/t_string.c
@@ -482,14 +482,13 @@ void strlenCommand(client *c) {
 
 /* LCS -- Longest common subsequence.
  *
- * LCS [IDX] [STOREIDX <key>] [MINMATCHLEN <len>]
+ * LCS [IDX] [MINMATCHLEN <len>]
  *     STRINGS <string> <string> | KEYS <keya> <keyb> */
 void lcsCommand(client *c) {
     uint32_t i, j;
     long long minmatchlen = 0;
     sds a = NULL, b = NULL;
     int getlen = 0, getidx = 0, withmatchlen = 0;
-    robj *idxkey = NULL; /* STOREIDX will set this and getidx to 1. */
     robj *obja = NULL, *objb = NULL;
 
     for (j = 1; j < (uint32_t)c->argc; j++) {
@@ -502,17 +501,16 @@ void lcsCommand(client *c) {
             getlen = 1;
         } else if (!strcasecmp(opt,"WITHMATCHLEN")) {
             withmatchlen = 1;
-        } else if (!strcasecmp(opt,"STOREIDX") && moreargs) {
-            getidx = 1;
-            idxkey = c->argv[j+1];
-            j++;
         } else if (!strcasecmp(opt,"MINMATCHLEN") && moreargs) {
             if (getLongLongFromObjectOrReply(c,c->argv[j+1],&minmatchlen,NULL)
                 != C_OK) return;
             if (minmatchlen < 0) minmatchlen = 0;
             j++;
         } else if (!strcasecmp(opt,"STRINGS")) {
-            if (moreargs != 2) {
+            if (a != NULL) {
+                addReplyError(c,"Either use STRINGS or KEYS");
+                return;
+            } else if (moreargs != 2) {
                 addReplyError(c,"LCS requires exactly two strings");
                 return;
             }
@@ -520,7 +518,10 @@ void lcsCommand(client *c) {
             b = c->argv[j+2]->ptr;
             j += 2;
         } else if (!strcasecmp(opt,"KEYS")) {
-            if (moreargs != 2) {
+            if (a != NULL) {
+                addReplyError(c,"Either use STRINGS or KEYS");
+                return;
+            } else if (moreargs != 2) {
                 addReplyError(c,"LCS requires exactly two keys");
                 return;
             }
@@ -542,12 +543,10 @@ void lcsCommand(client *c) {
         addReplyError(c,"Please specify two strings: "
                         "STRINGS or KEYS options are mandatory");
         return;
-    } else if (getlen && (getidx && idxkey == NULL)) {
+    } else if (getlen && getidx) {
         addReplyError(c,
-            "If you want both the LEN and indexes, please "
-            "store the indexes into a key with STOREIDX. However note "
-            "that the IDX output also includes both the LCS string and "
-            "its length");
+            "If you want both the length and indexes, please "
+            "just use IDX.");
         return;
     }
 
@@ -602,7 +601,7 @@ void lcsCommand(client *c) {
 
     /* Start with a deferred array if we have to emit the ranges. */
     uint32_t arraylen = 0;  /* Number of ranges emitted in the array. */
-    if (getidx && idxkey == NULL) {
+    if (getidx) {
         addReplyMapLen(c,2);
         addReplyBulkCString(c,"matches");
         arraylenptr = addReplyDeferredLen(c);

From 0b16f8d4449acf7a067de9922a7d3cd03563b86a Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Mon, 6 Apr 2020 13:45:37 +0200
Subject: [PATCH 034/186] LCS tests.

---
 tests/unit/type/string.tcl | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/tests/unit/type/string.tcl b/tests/unit/type/string.tcl
index 7122fd987..131a80ad0 100644
--- a/tests/unit/type/string.tcl
+++ b/tests/unit/type/string.tcl
@@ -419,4 +419,26 @@ start_server {tags {"string"}} {
         r set foo bar
         r getrange foo 0 4294967297
     } {bar}
+
+    set rna1 {CACCTTCCCAGGTAACAAACCAACCAACTTTCGATCTCTTGTAGATCTGTTCTCTAAACGAACTTTAAAATCTGTGTGGCTGTCACTCGGCTGCATGCTTAGTGCACTCACGCAGTATAATTAATAACTAATTACTGTCGTTGACAGGACACGAGTAACTCGTCTATCTTCTGCAGGCTGCTTACGGTTTCGTCCGTGTTGCAGCCGATCATCAGCACATCTAGGTTTCGTCCGGGTGTG}
+    set rna2 {ATTAAAGGTTTATACCTTCCCAGGTAACAAACCAACCAACTTTCGATCTCTTGTAGATCTGTTCTCTAAACGAACTTTAAAATCTGTGTGGCTGTCACTCGGCTGCATGCTTAGTGCACTCACGCAGTATAATTAATAACTAATTACTGTCGTTGACAGGACACGAGTAACTCGTCTATCTTCTGCAGGCTGCTTACGGTTTCGTCCGTGTTGCAGCCGATCATCAGCACATCTAGGTTT}
+    set rnalcs {ACCTTCCCAGGTAACAAACCAACCAACTTTCGATCTCTTGTAGATCTGTTCTCTAAACGAACTTTAAAATCTGTGTGGCTGTCACTCGGCTGCATGCTTAGTGCACTCACGCAGTATAATTAATAACTAATTACTGTCGTTGACAGGACACGAGTAACTCGTCTATCTTCTGCAGGCTGCTTACGGTTTCGTCCGTGTTGCAGCCGATCATCAGCACATCTAGGTTT}
+
+    test {LCS string output with STRINGS option} {
+        r LCS STRINGS $rna1 $rna2
+    } $rnalcs
+
+    test {LCS len} {
+        r LCS LEN STRINGS $rna1 $rna2
+    } [string length $rnalcs]
+
+    test {LCS with KEYS option} {
+        r set virus1 $rna1
+        r set virus2 $rna2
+        r LCS KEYS virus1 virus2
+    } $rnalcs
+
+    test {LCS indexes} {
+        dict get [r LCS IDX KEYS virus1 virus2] matches
+    } {{{238 238} {239 239}} {{236 236} {238 238}} {{229 230} {236 237}} {{224 224} {235 235}} {{1 222} {13 234}}}
 }

From c89e1f29373806b2d181edab81efcfb9cfc0403e Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Mon, 6 Apr 2020 13:48:31 +0200
Subject: [PATCH 035/186] LCS: allow KEYS / STRINGS to be anywhere.

Initially they needed to be at the end so that we could extend to N
strings in the future, but after further consideration I no longer
believe it's worth it.
---
 src/t_string.c | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/src/t_string.c b/src/t_string.c
index 39eb055fb..335bda404 100644
--- a/src/t_string.c
+++ b/src/t_string.c
@@ -510,9 +510,6 @@ void lcsCommand(client *c) {
             if (a != NULL) {
                 addReplyError(c,"Either use STRINGS or KEYS");
                 return;
-            } else if (moreargs != 2) {
-                addReplyError(c,"LCS requires exactly two strings");
-                return;
             }
             a = c->argv[j+1]->ptr;
             b = c->argv[j+2]->ptr;
@@ -521,9 +518,6 @@ void lcsCommand(client *c) {
             if (a != NULL) {
                 addReplyError(c,"Either use STRINGS or KEYS");
                 return;
-            } else if (moreargs != 2) {
-                addReplyError(c,"LCS requires exactly two keys");
-                return;
             }
             obja = lookupKeyRead(c->db,c->argv[j+1]);
             objb = lookupKeyRead(c->db,c->argv[j+2]);

From 5719b3054a534e62c25ae97680ecd4f7238ba484 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Mon, 6 Apr 2020 13:51:49 +0200
Subject: [PATCH 036/186] LCS: more tests.

---
 tests/unit/type/string.tcl | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/tests/unit/type/string.tcl b/tests/unit/type/string.tcl
index 131a80ad0..b9ef9de7a 100644
--- a/tests/unit/type/string.tcl
+++ b/tests/unit/type/string.tcl
@@ -441,4 +441,12 @@ start_server {tags {"string"}} {
     test {LCS indexes} {
         dict get [r LCS IDX KEYS virus1 virus2] matches
     } {{{238 238} {239 239}} {{236 236} {238 238}} {{229 230} {236 237}} {{224 224} {235 235}} {{1 222} {13 234}}}
+
+    test {LCS indexes with match len} {
+        dict get [r LCS IDX KEYS virus1 virus2 WITHMATCHLEN] matches
+    } {{{238 238} {239 239} 1} {{236 236} {238 238} 1} {{229 230} {236 237} 2} {{224 224} {235 235} 1} {{1 222} {13 234} 222}}
+
+    test {LCS indexes with match len and minimum match len} {
+        dict get [r LCS IDX KEYS virus1 virus2 WITHMATCHLEN MINMATCHLEN 5] matches
+    } {{{1 222} {13 234} 222}}
 }

From abd5156f259ace9dd5923d51da4646ba39c1e29f Mon Sep 17 00:00:00 2001
From: "zhaozhao.zz" <zhaozhao.zz@alibaba-inc.com>
Date: Wed, 17 Jul 2019 11:00:51 +0800
Subject: [PATCH 037/186] lazyfree: add a new configuration
 lazyfree-lazy-user-del

Delete keys in async way when executing DEL command, if
lazyfree-lazy-user-del is yes.
---
 redis.conf   | 5 ++++-
 src/config.c | 1 +
 src/db.c     | 2 +-
 src/server.h | 1 +
 4 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/redis.conf b/redis.conf
index 7c55a3ab0..14388e320 100644
--- a/redis.conf
+++ b/redis.conf
@@ -936,7 +936,9 @@ replica-priority 100
 #    or SORT with STORE option may delete existing keys. The SET command
 #    itself removes any old content of the specified key in order to replace
 #    it with the specified string.
-# 4) During replication, when a replica performs a full resynchronization with
+# 4) The DEL command itself, and normally it's not easy to replace DEL with
+#    UNLINK in user's codes.
+# 5) During replication, when a replica performs a full resynchronization with
 #    its master, the content of the whole database is removed in order to
 #    load the RDB file just transferred.
 #
@@ -948,6 +950,7 @@ replica-priority 100
 lazyfree-lazy-eviction no
 lazyfree-lazy-expire no
 lazyfree-lazy-server-del no
+lazyfree-lazy-user-del no
 replica-lazy-flush no
 
 ################################ THREADED I/O #################################
diff --git a/src/config.c b/src/config.c
index 7c87ebe6e..e0cbcc281 100644
--- a/src/config.c
+++ b/src/config.c
@@ -2099,6 +2099,7 @@ standardConfig configs[] = {
     createBoolConfig("lazyfree-lazy-eviction", NULL, MODIFIABLE_CONFIG, server.lazyfree_lazy_eviction, 0, NULL, NULL),
     createBoolConfig("lazyfree-lazy-expire", NULL, MODIFIABLE_CONFIG, server.lazyfree_lazy_expire, 0, NULL, NULL),
     createBoolConfig("lazyfree-lazy-server-del", NULL, MODIFIABLE_CONFIG, server.lazyfree_lazy_server_del, 0, NULL, NULL),
+    createBoolConfig("lazyfree-lazy-user-del", NULL, MODIFIABLE_CONFIG, server.lazyfree_lazy_user_del , 0, NULL, NULL),
     createBoolConfig("repl-disable-tcp-nodelay", NULL, MODIFIABLE_CONFIG, server.repl_disable_tcp_nodelay, 0, NULL, NULL),
     createBoolConfig("repl-diskless-sync", NULL, MODIFIABLE_CONFIG, server.repl_diskless_sync, 0, NULL, NULL),
     createBoolConfig("gopher-enabled", NULL, MODIFIABLE_CONFIG, server.gopher_enabled, 0, NULL, NULL),
diff --git a/src/db.c b/src/db.c
index 6e5a8bf3a..9b5d62f29 100644
--- a/src/db.c
+++ b/src/db.c
@@ -556,7 +556,7 @@ void delGenericCommand(client *c, int lazy) {
 }
 
 void delCommand(client *c) {
-    delGenericCommand(c,0);
+    delGenericCommand(c,server.lazyfree_lazy_user_del);
 }
 
 void unlinkCommand(client *c) {
diff --git a/src/server.h b/src/server.h
index b17995948..9b77f55ac 100644
--- a/src/server.h
+++ b/src/server.h
@@ -1397,6 +1397,7 @@ struct redisServer {
     int lazyfree_lazy_eviction;
     int lazyfree_lazy_expire;
     int lazyfree_lazy_server_del;
+    int lazyfree_lazy_user_del;
     /* Latency monitor */
     long long latency_monitor_threshold;
     dict *latency_events;

From c8dbcff9dbe724dfa0c369a289e6f2f6dda13cab Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Mon, 6 Apr 2020 17:32:04 +0200
Subject: [PATCH 038/186] Clarify redis.conf comment about
 lazyfree-lazy-user-del.

---
 redis.conf | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/redis.conf b/redis.conf
index 14388e320..5baeae65f 100644
--- a/redis.conf
+++ b/redis.conf
@@ -936,23 +936,27 @@ replica-priority 100
 #    or SORT with STORE option may delete existing keys. The SET command
 #    itself removes any old content of the specified key in order to replace
 #    it with the specified string.
-# 4) The DEL command itself, and normally it's not easy to replace DEL with
-#    UNLINK in user's codes.
-# 5) During replication, when a replica performs a full resynchronization with
+# 4) During replication, when a replica performs a full resynchronization with
 #    its master, the content of the whole database is removed in order to
 #    load the RDB file just transferred.
 #
 # In all the above cases the default is to delete objects in a blocking way,
 # like if DEL was called. However you can configure each case specifically
 # in order to instead release memory in a non-blocking way like if UNLINK
-# was called, using the following configuration directives:
+# was called, using the following configuration directives.
 
 lazyfree-lazy-eviction no
 lazyfree-lazy-expire no
 lazyfree-lazy-server-del no
-lazyfree-lazy-user-del no
 replica-lazy-flush no
 
+# It is also possible, for the case when to replace the user code DEL calls
+# with UNLINK calls is not easy, to modify the default behavior of the DEL
+# command to act exactly like UNLINK, using the following configuration
+# directive:
+
+lazyfree-lazy-user-del no
+
 ################################ THREADED I/O #################################
 
 # Redis is mostly single threaded, however there are certain threaded

From c3ac717487099ef8b3a9a47a2f8f147b3230b3ba Mon Sep 17 00:00:00 2001
From: qetu3790 <qetu3790@163.com>
Date: Mon, 6 Apr 2020 20:52:32 +0800
Subject: [PATCH 039/186] fix comments about RESIZE DB opcode in rdb.c

fix comments about RESIZE DB opcode in rdb.c
---
 src/rdb.c | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/src/rdb.c b/src/rdb.c
index fc8911979..78ec83cce 100644
--- a/src/rdb.c
+++ b/src/rdb.c
@@ -1175,10 +1175,7 @@ int rdbSaveRio(rio *rdb, int *error, int rdbflags, rdbSaveInfo *rsi) {
         if (rdbSaveType(rdb,RDB_OPCODE_SELECTDB) == -1) goto werr;
         if (rdbSaveLen(rdb,j) == -1) goto werr;
 
-        /* Write the RESIZE DB opcode. We trim the size to UINT32_MAX, which
-         * is currently the largest type we are able to represent in RDB sizes.
-         * However this does not limit the actual size of the DB to load since
-         * these sizes are just hints to resize the hash tables. */
+        /* Write the RESIZE DB opcode. */
         uint64_t db_size, expires_size;
         db_size = dictSize(db->dict);
         expires_size = dictSize(db->expires);

From 35c64b898c1e4a5c2809888671730156731367a2 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Tue, 7 Apr 2020 12:07:09 +0200
Subject: [PATCH 040/186] Speedup INFO by counting client memory incrementally.

Related to #5145.

Design note: clients may change type when they turn into replicas or are
moved into the Pub/Sub category and so forth. Moreover the recomputation
of the bytes used is problematic for obvious reasons: it changes
continuously, so as a conservative way to avoid accumulating errors,
each client remembers the contribution it gave to the sum, and removes
it when it is freed or before updating it with the new memory usage.
---
 src/networking.c |  7 +++++++
 src/object.c     | 33 +++++++++------------------------
 src/server.c     | 25 +++++++++++++++++++++++++
 src/server.h     | 13 +++++++++++--
 4 files changed, 52 insertions(+), 26 deletions(-)

diff --git a/src/networking.c b/src/networking.c
index 85c640e34..654fda517 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -157,6 +157,8 @@ client *createClient(connection *conn) {
     c->client_list_node = NULL;
     c->client_tracking_redirection = 0;
     c->client_tracking_prefixes = NULL;
+    c->client_cron_last_memory_usage = 0;
+    c->client_cron_last_memory_type = CLIENT_TYPE_NORMAL;
     c->auth_callback = NULL;
     c->auth_callback_privdata = NULL;
     c->auth_module = NULL;
@@ -1160,6 +1162,11 @@ void freeClient(client *c) {
         listDelNode(server.clients_to_close,ln);
     }
 
+    /* Remove the contribution that this client gave to our
+     * incrementally computed memory usage. */
+    server.stat_clients_type_memory[c->client_cron_last_memory_type] -=
+        c->client_cron_last_memory_usage;
+
     /* Release other dynamically allocated client structure fields,
      * and finally release the client structure itself. */
     if (c->name) decrRefCount(c->name);
diff --git a/src/object.c b/src/object.c
index 11e335afc..52d5b11f5 100644
--- a/src/object.c
+++ b/src/object.c
@@ -974,30 +974,15 @@ struct redisMemOverhead *getMemoryOverheadData(void) {
     mh->repl_backlog = mem;
     mem_total += mem;
 
-    mem = 0;
-    if (listLength(server.clients)) {
-        listIter li;
-        listNode *ln;
-        size_t mem_normal = 0, mem_slaves = 0;
-
-        listRewind(server.clients,&li);
-        while((ln = listNext(&li))) {
-            size_t mem_curr = 0;
-            client *c = listNodeValue(ln);
-            int type = getClientType(c);
-            mem_curr += getClientOutputBufferMemoryUsage(c);
-            mem_curr += sdsAllocSize(c->querybuf);
-            mem_curr += sizeof(client);
-            if (type == CLIENT_TYPE_SLAVE)
-                mem_slaves += mem_curr;
-            else
-                mem_normal += mem_curr;
-        }
-        mh->clients_slaves = mem_slaves;
-        mh->clients_normal = mem_normal;
-        mem = mem_slaves + mem_normal;
-    }
-    mem_total+=mem;
+    /* Computing the memory used by the clients would be O(N) if done
+     * here online. We use our values computed incrementally by
+     * clientsCronTrackClientsMemUsage(). */
+    mh->clients_slaves = server.stat_clients_type_memory[CLIENT_TYPE_SLAVE];
+    mh->clients_normal = server.stat_clients_type_memory[CLIENT_TYPE_MASTER]+
+                         server.stat_clients_type_memory[CLIENT_TYPE_PUBSUB]+
+                         server.stat_clients_type_memory[CLIENT_TYPE_NORMAL];
+    mem_total += mh->clients_slaves;
+    mem_total += mh->clients_normal;
 
     mem = 0;
     if (server.aof_state != AOF_OFF) {
diff --git a/src/server.c b/src/server.c
index 56feb09a3..996e0f5d2 100644
--- a/src/server.c
+++ b/src/server.c
@@ -1593,6 +1593,28 @@ int clientsCronTrackExpansiveClients(client *c) {
     return 0; /* This function never terminates the client. */
 }
 
+/* Iterating all the clients in getMemoryOverheadData() is too slow and
+ * in turn would make the INFO command too slow. So we perform this
+ * computation incrementally and track the (not instantaneous but updated
+ * to the second) total memory used by clients using clinetsCron() in
+ * a more incremental way (depending on server.hz). */
+int clientsCronTrackClientsMemUsage(client *c) {
+    size_t mem = 0;
+    int type = getClientType(c);
+    mem += getClientOutputBufferMemoryUsage(c);
+    mem += sdsAllocSize(c->querybuf);
+    mem += sizeof(client);
+    /* Now that we have the memory used by the client, remove the old
+     * value from the old categoty, and add it back. */
+    server.stat_clients_type_memory[c->client_cron_last_memory_type] -=
+        c->client_cron_last_memory_usage;
+    server.stat_clients_type_memory[type] += mem;
+    /* Remember what we added and where, to remove it next time. */
+    c->client_cron_last_memory_usage = mem;
+    c->client_cron_last_memory_type = type;
+    return 0;
+}
+
 /* Return the max samples in the memory usage of clients tracked by
  * the function clientsCronTrackExpansiveClients(). */
 void getExpansiveClientsInfo(size_t *in_usage, size_t *out_usage) {
@@ -1653,6 +1675,7 @@ void clientsCron(void) {
         if (clientsCronHandleTimeout(c,now)) continue;
         if (clientsCronResizeQueryBuffer(c)) continue;
         if (clientsCronTrackExpansiveClients(c)) continue;
+        if (clientsCronTrackClientsMemUsage(c)) continue;
     }
 }
 
@@ -2792,6 +2815,8 @@ void initServer(void) {
     server.stat_rdb_cow_bytes = 0;
     server.stat_aof_cow_bytes = 0;
     server.stat_module_cow_bytes = 0;
+    for (int j = 0; j < CLIENT_TYPE_COUNT; j++)
+        server.stat_clients_type_memory[j] = 0;
     server.cron_malloc_stats.zmalloc_used = 0;
     server.cron_malloc_stats.process_rss = 0;
     server.cron_malloc_stats.allocator_allocated = 0;
diff --git a/src/server.h b/src/server.h
index 9b77f55ac..cf4c285f8 100644
--- a/src/server.h
+++ b/src/server.h
@@ -274,6 +274,7 @@ typedef long long ustime_t; /* microsecond time type. */
 #define CLIENT_TYPE_SLAVE 1  /* Slaves. */
 #define CLIENT_TYPE_PUBSUB 2 /* Clients subscribed to PubSub channels. */
 #define CLIENT_TYPE_MASTER 3 /* Master. */
+#define CLIENT_TYPE_COUNT 4  /* Total number of client types. */
 #define CLIENT_TYPE_OBUF_COUNT 3 /* Number of clients to expose to output
                                     buffer configuration. Just the first
                                     three: normal, slave, pubsub. */
@@ -820,10 +821,10 @@ typedef struct client {
                                                * when the authenticated user
                                                * changes. */
     void *auth_callback_privdata; /* Private data that is passed when the auth
-                                   * changed callback is executed. Opaque for 
+                                   * changed callback is executed. Opaque for
                                    * Redis Core. */
     void *auth_module;      /* The module that owns the callback, which is used
-                             * to disconnect the client if the module is 
+                             * to disconnect the client if the module is
                              * unloaded for cleanup. Opaque for Redis Core.*/
 
     /* If this client is in tracking mode and this field is non zero,
@@ -833,6 +834,13 @@ typedef struct client {
     rax *client_tracking_prefixes; /* A dictionary of prefixes we are already
                                       subscribed to in BCAST mode, in the
                                       context of client side caching. */
+    /* In clientsCronTrackClientsMemUsage() we track the memory usage of
+     * each client and add it to the sum of all the clients of a given type,
+     * however we need to remember what was the old contribution of each
+     * client, and in which categoty the client was, in order to remove it
+     * before adding it the new value. */
+    uint64_t client_cron_last_memory_usage;
+    int      client_cron_last_memory_type;
     /* Response buffer */
     int bufpos;
     char buf[PROTO_REPLY_CHUNK_BYTES];
@@ -1129,6 +1137,7 @@ struct redisServer {
     size_t stat_rdb_cow_bytes;      /* Copy on write bytes during RDB saving. */
     size_t stat_aof_cow_bytes;      /* Copy on write bytes during AOF rewrite. */
     size_t stat_module_cow_bytes;   /* Copy on write bytes during module fork. */
+    uint64_t stat_clients_type_memory[CLIENT_TYPE_COUNT];/* Mem usage by type */
     long long stat_unexpected_error_replies; /* Number of unexpected (aof-loading, replica to master, etc.) error replies */
     /* The following two are used to track instantaneous metrics, like
      * number of operations per second, network traffic. */

From 2437455f2769be96e948c8b0ffed10ef684f160d Mon Sep 17 00:00:00 2001
From: mymilkbottles <javaxz@163.com>
Date: Mon, 6 Apr 2020 19:27:06 +0800
Subject: [PATCH 041/186] Judge the log level in advance

---
 src/scripting.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/scripting.c b/src/scripting.c
index 32a511e13..bccbcf637 100644
--- a/src/scripting.c
+++ b/src/scripting.c
@@ -959,6 +959,7 @@ int luaLogCommand(lua_State *lua) {
         lua_pushstring(lua, "Invalid debug level.");
         return lua_error(lua);
     }
+    if (level < server.verbosity) return 0;
 
     /* Glue together all the arguments */
     log = sdsempty();

From cbcd07777dc569618a34f59e5fd0de53178f4f1d Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Wed, 8 Apr 2020 10:54:18 +0200
Subject: [PATCH 042/186] Fix ACL HELP table missing comma.

---
 src/acl.c | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/src/acl.c b/src/acl.c
index 27f4bdb84..733988013 100644
--- a/src/acl.c
+++ b/src/acl.c
@@ -1847,18 +1847,18 @@ void aclCommand(client *c) {
         }
     } else if (!strcasecmp(sub,"help")) {
         const char *help[] = {
-"LOAD                              -- Reload users from the ACL file.",
-"SAVE                              -- Save the current config to the ACL file."
-"LIST                              -- Show user details in config file format.",
-"USERS                             -- List all the registered usernames.",
-"SETUSER <username> [attribs ...]  -- Create or modify a user.",
-"GETUSER <username>                -- Get the user details.",
-"DELUSER <username> [...]          -- Delete a list of users.",
-"CAT                               -- List available categories.",
-"CAT <category>                    -- List commands inside category.",
-"GENPASS                           -- Generate a secure user password.",
-"WHOAMI                            -- Return the current connection username.",
-"LOG [<count> | RESET]             -- Show the ACL log entries.",
+"LOAD                             -- Reload users from the ACL file.",
+"SAVE                             -- Save the current config to the ACL file.",
+"LIST                             -- Show user details in config file format.",
+"USERS                            -- List all the registered usernames.",
+"SETUSER <username> [attribs ...] -- Create or modify a user.",
+"GETUSER <username>               -- Get the user details.",
+"DELUSER <username> [...]         -- Delete a list of users.",
+"CAT                              -- List available categories.",
+"CAT <category>                   -- List commands inside category.",
+"GENPASS                          -- Generate a secure user password.",
+"WHOAMI                           -- Return the current connection username.",
+"LOG [<count> | RESET]            -- Show the ACL log entries.",
 NULL
         };
         addReplyHelp(c,help);

From bde1f0a8e2a755f4090d2b57048c3887da7b9e8e Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Fri, 10 Apr 2020 10:12:26 +0200
Subject: [PATCH 043/186] RESP3: change streams items from maps to arrays.

Streams items are similar to dictionaries, however they preserve both
the order, and allow for duplicated field names. So a map is not a
semantically sounding way to deal with this.

https://twitter.com/antirez/status/1248261087553880069
---
 src/t_stream.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/t_stream.c b/src/t_stream.c
index e0af87f97..4ce3a9b25 100644
--- a/src/t_stream.c
+++ b/src/t_stream.c
@@ -963,7 +963,7 @@ size_t streamReplyWithRange(client *c, stream *s, streamID *start, streamID *end
         addReplyArrayLen(c,2);
         addReplyStreamID(c,&id);
 
-        addReplyMapLen(c,numfields);
+        addReplyArrayLen(c,numfields*2);
 
         /* Emit the field-value pairs. */
         while(numfields--) {

From ddeda9ceb777bf09c0139cc57bf021757e98073c Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Tue, 14 Apr 2020 10:52:40 +0200
Subject: [PATCH 044/186] Fix function names in zslDeleteNode() top comment.

---
 src/t_zset.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/t_zset.c b/src/t_zset.c
index ea6f4b848..147313d53 100644
--- a/src/t_zset.c
+++ b/src/t_zset.c
@@ -186,7 +186,8 @@ zskiplistNode *zslInsert(zskiplist *zsl, double score, sds ele) {
     return x;
 }
 
-/* Internal function used by zslDelete, zslDeleteByScore and zslDeleteByRank */
+/* Internal function used by zslDelete, zslDeleteRangeByScore and
+ * zslDeleteRangeByRank. */
 void zslDeleteNode(zskiplist *zsl, zskiplistNode *x, zskiplistNode **update) {
     int i;
     for (i = 0; i < zsl->level; i++) {

From 8f5157058b54de6a81ec4b55cf72b401ab562035 Mon Sep 17 00:00:00 2001
From: hayleeliu <liuhuailing_123@163.com>
Date: Wed, 8 Apr 2020 18:20:32 +0800
Subject: [PATCH 045/186] fix spelling mistake in bitops.c

---
 src/bitops.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/bitops.c b/src/bitops.c
index f78e4fd34..496d78b66 100644
--- a/src/bitops.c
+++ b/src/bitops.c
@@ -269,7 +269,7 @@ int64_t getSignedBitfield(unsigned char *p, uint64_t offset, uint64_t bits) {
  * then zero is returned, otherwise in case of overflow, 1 is returned,
  * otherwise in case of underflow, -1 is returned.
  *
- * When non-zero is returned (oferflow or underflow), if not NULL, *limit is
+ * When non-zero is returned (overflow or underflow), if not NULL, *limit is
  * set to the value the operation should result when an overflow happens,
  * depending on the specified overflow semantics:
  *

From 61b153073fe73a13d08e5920af90036b1c47ec9e Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Thu, 9 Apr 2020 10:24:10 +0200
Subject: [PATCH 046/186] RDB: load files faster avoiding useless free+realloc.

Reloading of the RDB generated by

    DEBUG POPULATE 5000000
    SAVE

is now 25% faster.

This commit also prepares the ability to have more flexibility when
loading stuff from the RDB, since we no longer use dbAdd() but can
control exactly how things are added in the database.
---
 src/cluster.c         |  2 +-
 src/db.c              | 16 ++++++++--------
 src/lazyfree.c        |  2 +-
 src/rdb.c             | 40 ++++++++++++++++++++++++++--------------
 src/rdb.h             |  2 +-
 src/redis-check-rdb.c |  2 +-
 src/server.h          |  4 ++--
 7 files changed, 40 insertions(+), 28 deletions(-)

diff --git a/src/cluster.c b/src/cluster.c
index 385ff5763..2377b386b 100644
--- a/src/cluster.c
+++ b/src/cluster.c
@@ -4966,7 +4966,7 @@ void restoreCommand(client *c) {
 
     rioInitWithBuffer(&payload,c->argv[3]->ptr);
     if (((type = rdbLoadObjectType(&payload)) == -1) ||
-        ((obj = rdbLoadObject(type,&payload,c->argv[1])) == NULL))
+        ((obj = rdbLoadObject(type,&payload,c->argv[1]->ptr)) == NULL))
     {
         addReplyError(c,"Bad data format");
         return;
diff --git a/src/db.c b/src/db.c
index 9b5d62f29..d393a5fdd 100644
--- a/src/db.c
+++ b/src/db.c
@@ -185,7 +185,7 @@ void dbAdd(redisDb *db, robj *key, robj *val) {
         val->type == OBJ_ZSET ||
         val->type == OBJ_STREAM)
         signalKeyAsReady(db, key);
-    if (server.cluster_enabled) slotToKeyAdd(key);
+    if (server.cluster_enabled) slotToKeyAdd(key->ptr);
 }
 
 /* Overwrite an existing key with a new value. Incrementing the reference
@@ -288,7 +288,7 @@ int dbSyncDelete(redisDb *db, robj *key) {
      * the key, because it is shared with the main dictionary. */
     if (dictSize(db->expires) > 0) dictDelete(db->expires,key->ptr);
     if (dictDelete(db->dict,key->ptr) == DICT_OK) {
-        if (server.cluster_enabled) slotToKeyDel(key);
+        if (server.cluster_enabled) slotToKeyDel(key->ptr);
         return 1;
     } else {
         return 0;
@@ -1647,17 +1647,17 @@ int *xreadGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numkeys)
  * a fast way a key that belongs to a specified hash slot. This is useful
  * while rehashing the cluster and in other conditions when we need to
  * understand if we have keys for a given hash slot. */
-void slotToKeyUpdateKey(robj *key, int add) {
-    unsigned int hashslot = keyHashSlot(key->ptr,sdslen(key->ptr));
+void slotToKeyUpdateKey(sds key, int add) {
+    size_t keylen = sdslen(key);
+    unsigned int hashslot = keyHashSlot(key,keylen);
     unsigned char buf[64];
     unsigned char *indexed = buf;
-    size_t keylen = sdslen(key->ptr);
 
     server.cluster->slots_keys_count[hashslot] += add ? 1 : -1;
     if (keylen+2 > 64) indexed = zmalloc(keylen+2);
     indexed[0] = (hashslot >> 8) & 0xff;
     indexed[1] = hashslot & 0xff;
-    memcpy(indexed+2,key->ptr,keylen);
+    memcpy(indexed+2,key,keylen);
     if (add) {
         raxInsert(server.cluster->slots_to_keys,indexed,keylen+2,NULL,NULL);
     } else {
@@ -1666,11 +1666,11 @@ void slotToKeyUpdateKey(robj *key, int add) {
     if (indexed != buf) zfree(indexed);
 }
 
-void slotToKeyAdd(robj *key) {
+void slotToKeyAdd(sds key) {
     slotToKeyUpdateKey(key,1);
 }
 
-void slotToKeyDel(robj *key) {
+void slotToKeyDel(sds key) {
     slotToKeyUpdateKey(key,0);
 }
 
diff --git a/src/lazyfree.c b/src/lazyfree.c
index 3d3159c90..f01504e70 100644
--- a/src/lazyfree.c
+++ b/src/lazyfree.c
@@ -83,7 +83,7 @@ int dbAsyncDelete(redisDb *db, robj *key) {
      * field to NULL in order to lazy free it later. */
     if (de) {
         dictFreeUnlinkedEntry(db->dict,de);
-        if (server.cluster_enabled) slotToKeyDel(key);
+        if (server.cluster_enabled) slotToKeyDel(key->ptr);
         return 1;
     } else {
         return 0;
diff --git a/src/rdb.c b/src/rdb.c
index 78ec83cce..d3104ffbf 100644
--- a/src/rdb.c
+++ b/src/rdb.c
@@ -1422,7 +1422,7 @@ robj *rdbLoadCheckModuleValue(rio *rdb, char *modulename) {
 
 /* Load a Redis object of the specified type from the specified file.
  * On success a newly allocated object is returned, otherwise NULL. */
-robj *rdbLoadObject(int rdbtype, rio *rdb, robj *key) {
+robj *rdbLoadObject(int rdbtype, rio *rdb, sds key) {
     robj *o = NULL, *ele, *dec;
     uint64_t len;
     unsigned int i;
@@ -1886,7 +1886,9 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, robj *key) {
             exit(1);
         }
         RedisModuleIO io;
-        moduleInitIOContext(io,mt,rdb,key);
+        robj keyobj;
+        initStaticStringObject(keyobj,key);
+        moduleInitIOContext(io,mt,rdb,&keyobj);
         io.ver = (rdbtype == RDB_TYPE_MODULE) ? 1 : 2;
         /* Call the rdb_load method of the module providing the 10 bit
          * encoding version in the lower 10 bits of the module ID. */
@@ -2044,7 +2046,8 @@ int rdbLoadRio(rio *rdb, int rdbflags, rdbSaveInfo *rsi) {
     long long lru_clock = LRU_CLOCK();
 
     while(1) {
-        robj *key, *val;
+        sds key;
+        robj *val;
 
         /* Read type. */
         if ((type = rdbLoadType(rdb)) == -1) goto eoferr;
@@ -2216,10 +2219,11 @@ int rdbLoadRio(rio *rdb, int rdbflags, rdbSaveInfo *rsi) {
         }
 
         /* Read key */
-        if ((key = rdbLoadStringObject(rdb)) == NULL) goto eoferr;
+        if ((key = rdbGenericLoadStringObject(rdb,RDB_LOAD_SDS,NULL)) == NULL)
+            goto eoferr;
         /* Read value */
         if ((val = rdbLoadObject(type,rdb,key)) == NULL) {
-            decrRefCount(key);
+            sdsfree(key);
             goto eoferr;
         }
 
@@ -2229,24 +2233,32 @@ int rdbLoadRio(rio *rdb, int rdbflags, rdbSaveInfo *rsi) {
          * responsible for key expiry. If we would expire keys here, the
          * snapshot taken by the master may not be reflected on the slave. */
         if (iAmMaster() && !(rdbflags&RDBFLAGS_AOF_PREAMBLE) && expiretime != -1 && expiretime < now) {
-            decrRefCount(key);
+            sdsfree(key);
             decrRefCount(val);
         } else {
             /* Add the new object in the hash table */
-            dbAdd(db,key,val);
+            int retval = dictAdd(db->dict, key, val);
+            if (retval != DICT_OK) {
+                serverLog(LL_WARNING,
+                    "RDB has duplicated key '%s' in DB %d",key,db->id);
+                serverPanic("Duplicated key found in RDB file");
+            }
+            if (server.cluster_enabled) slotToKeyAdd(key);
 
             /* Set the expire time if needed */
-            if (expiretime != -1) setExpire(NULL,db,key,expiretime);
+            if (expiretime != -1) {
+                robj keyobj;
+                initStaticStringObject(keyobj,key);
+                setExpire(NULL,db,&keyobj,expiretime);
+            }
 
             /* Set usage information (for eviction). */
             objectSetLRUOrLFU(val,lfu_freq,lru_idle,lru_clock,1000);
-
-            /* Decrement the key refcount since dbAdd() will take its
-             * own reference. */
-            decrRefCount(key);
         }
-        if (server.key_load_delay)
-            usleep(server.key_load_delay);
+
+        /* Loading the database more slowly is useful in order to test
+         * certain edge cases. */
+        if (server.key_load_delay) usleep(server.key_load_delay);
 
         /* Reset the state that is key-specified and is populated by
          * opcodes before the key, so that we start from scratch again. */
diff --git a/src/rdb.h b/src/rdb.h
index b276a978b..526ed116b 100644
--- a/src/rdb.h
+++ b/src/rdb.h
@@ -144,7 +144,7 @@ void rdbRemoveTempFile(pid_t childpid);
 int rdbSave(char *filename, rdbSaveInfo *rsi);
 ssize_t rdbSaveObject(rio *rdb, robj *o, robj *key);
 size_t rdbSavedObjectLen(robj *o, robj *key);
-robj *rdbLoadObject(int type, rio *rdb, robj *key);
+robj *rdbLoadObject(int type, rio *rdb, sds key);
 void backgroundSaveDoneHandler(int exitcode, int bysignal);
 int rdbSaveKeyValuePair(rio *rdb, robj *key, robj *val, long long expiretime);
 ssize_t rdbSaveSingleModuleAux(rio *rdb, int when, moduleType *mt);
diff --git a/src/redis-check-rdb.c b/src/redis-check-rdb.c
index 1210d49b4..17ec656ce 100644
--- a/src/redis-check-rdb.c
+++ b/src/redis-check-rdb.c
@@ -287,7 +287,7 @@ int redis_check_rdb(char *rdbfilename, FILE *fp) {
         rdbstate.keys++;
         /* Read value */
         rdbstate.doing = RDB_CHECK_DOING_READ_OBJECT_VALUE;
-        if ((val = rdbLoadObject(type,&rdb,key)) == NULL) goto eoferr;
+        if ((val = rdbLoadObject(type,&rdb,key->ptr)) == NULL) goto eoferr;
         /* Check if the key already expired. */
         if (expiretime != -1 && expiretime < now)
             rdbstate.already_expired++;
diff --git a/src/server.h b/src/server.h
index cf4c285f8..b8c46153d 100644
--- a/src/server.h
+++ b/src/server.h
@@ -2093,8 +2093,8 @@ unsigned int delKeysInSlot(unsigned int hashslot);
 int verifyClusterConfigWithData(void);
 void scanGenericCommand(client *c, robj *o, unsigned long cursor);
 int parseScanCursorOrReply(client *c, robj *o, unsigned long *cursor);
-void slotToKeyAdd(robj *key);
-void slotToKeyDel(robj *key);
+void slotToKeyAdd(sds key);
+void slotToKeyDel(sds key);
 void slotToKeyFlush(void);
 int dbAsyncDelete(redisDb *db, robj *key);
 void emptyDbAsync(redisDb *db);

From 8f896e57ae15c8ce24a0d417224b0eb17520f1ee Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Tue, 14 Apr 2020 11:23:44 +0200
Subject: [PATCH 047/186] Fix zsetAdd() top comment spelling.

---
 src/t_zset.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/t_zset.c b/src/t_zset.c
index 147313d53..5c000e76f 100644
--- a/src/t_zset.c
+++ b/src/t_zset.c
@@ -1301,14 +1301,14 @@ int zsetScore(robj *zobj, sds member, double *score) {
  * none could be set if we re-added an element using the same score it used
  * to have, or in the case a zero increment is used).
  *
- * The function returns 0 on erorr, currently only when the increment
+ * The function returns 0 on error, currently only when the increment
  * produces a NAN condition, or when the 'score' value is NAN since the
  * start.
  *
- * The commad as a side effect of adding a new element may convert the sorted
+ * The command as a side effect of adding a new element may convert the sorted
  * set internal encoding from ziplist to hashtable+skiplist.
  *
- * Memory managemnet of 'ele':
+ * Memory management of 'ele':
  *
  * The function does not take ownership of the 'ele' SDS string, but copies
  * it if needed. */

From 8161a7a3efdffd8f09d379c6b8531f0110d983e1 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Thu, 9 Apr 2020 11:09:40 +0200
Subject: [PATCH 048/186] RDB: clarify a condition in rdbLoadRio().

---
 src/rdb.c | 10 ++++++++--
 src/rdb.h |  1 +
 2 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/src/rdb.c b/src/rdb.c
index d3104ffbf..313830c25 100644
--- a/src/rdb.c
+++ b/src/rdb.c
@@ -2231,8 +2231,14 @@ int rdbLoadRio(rio *rdb, int rdbflags, rdbSaveInfo *rsi) {
          * an RDB file from disk, either at startup, or when an RDB was
          * received from the master. In the latter case, the master is
          * responsible for key expiry. If we would expire keys here, the
-         * snapshot taken by the master may not be reflected on the slave. */
-        if (iAmMaster() && !(rdbflags&RDBFLAGS_AOF_PREAMBLE) && expiretime != -1 && expiretime < now) {
+         * snapshot taken by the master may not be reflected on the slave.
+         * Similarly if the RDB is the preamble of an AOF file, we want to
+         * load all the keys as they are, since the log of operations later
+         * assume to work in an exact keyspace state. */
+        if (iAmMaster() &&
+            !(rdbflags&RDBFLAGS_AOF_PREAMBLE) &&
+            expiretime != -1 && expiretime < now)
+        {
             sdsfree(key);
             decrRefCount(val);
         } else {
diff --git a/src/rdb.h b/src/rdb.h
index 526ed116b..9dfcae7a5 100644
--- a/src/rdb.h
+++ b/src/rdb.h
@@ -125,6 +125,7 @@
 #define RDBFLAGS_NONE 0
 #define RDBFLAGS_AOF_PREAMBLE (1<<0)
 #define RDBFLAGS_REPLICATION (1<<1)
+#define RDBFLAGS_ALLOW_DUP (1<<2)
 
 int rdbSaveType(rio *rdb, unsigned char type);
 int rdbLoadType(rio *rdb);

From b92d9a895f5d22994364b4867e90262e1fcbb5ac Mon Sep 17 00:00:00 2001
From: hwware <wen.hui.ware@gmail.com>
Date: Tue, 14 Apr 2020 00:16:29 -0400
Subject: [PATCH 049/186] fix spelling in acl.c

---
 src/acl.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/acl.c b/src/acl.c
index 733988013..6847130ad 100644
--- a/src/acl.c
+++ b/src/acl.c
@@ -1783,8 +1783,8 @@ void aclCommand(client *c) {
         long count = 10; /* Number of entries to emit by default. */
 
         /* Parse the only argument that LOG may have: it could be either
-         * the number of entires the user wants to display, or alternatively
-         * the "RESET" command in order to flush the old entires. */
+         * the number of entries the user wants to display, or alternatively
+         * the "RESET" command in order to flush the old entries. */
         if (c->argc == 3) {
             if (!strcasecmp(c->argv[2]->ptr,"reset")) {
                 listSetFreeMethod(ACLLog,ACLFreeLogEntry);

From 23094ba015382fe08293189c06be7f77326bfea8 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Thu, 9 Apr 2020 12:02:27 +0200
Subject: [PATCH 050/186] More powerful DEBUG RELOAD.

Related to #3243.
---
 src/debug.c | 45 +++++++++++++++++++++++++++++++++++++--------
 src/rdb.c   | 18 ++++++++++++++----
 src/rdb.h   |  8 ++++----
 3 files changed, 55 insertions(+), 16 deletions(-)

diff --git a/src/debug.c b/src/debug.c
index baaaa2424..1351b2536 100644
--- a/src/debug.c
+++ b/src/debug.c
@@ -366,7 +366,7 @@ void debugCommand(client *c) {
 "OOM -- Crash the server simulating an out-of-memory error.",
 "PANIC -- Crash the server simulating a panic.",
 "POPULATE <count> [prefix] [size] -- Create <count> string keys named key:<num>. If a prefix is specified is used instead of the 'key' prefix.",
-"RELOAD -- Save the RDB on disk and reload it back in memory.",
+"RELOAD [MERGE] [NOFLUSH] [NOSAVE] -- Save the RDB on disk and reload it back in memory. By default it will save the RDB file and load it back. With the NOFLUSH option the current database is not removed before loading the new one, but conficts in keys will kill the server with an exception. When MERGE is used, conflicting keys will be loaded (the key in the loaded RDB file will win). When NOSAVE is used, the server will not save the current dataset in the RDB file before loading. Use DEBUG RELOAD NOSAVE when you want just to load the RDB file you placed in the Redis working directory in order to replace the current dataset in memory. Use DEBUG RELOAD NOSAVE NOFLUSH MERGE when you want to add what is in the current RDB file placed in the Redis current directory, with the current memory content. Use DEBUG RELOAD when you want to verify Redis is able to persist the current dataset in the RDB file, flush the memory content, and load it back.",
 "RESTART -- Graceful restart: save config, db, restart.",
 "SDSLEN <key> -- Show low level SDS string info representing key and value.",
 "SEGFAULT -- Crash the server with sigsegv.",
@@ -411,15 +411,44 @@ NULL
         serverLog(LL_WARNING, "DEBUG LOG: %s", (char*)c->argv[2]->ptr);
         addReply(c,shared.ok);
     } else if (!strcasecmp(c->argv[1]->ptr,"reload")) {
-        rdbSaveInfo rsi, *rsiptr;
-        rsiptr = rdbPopulateSaveInfo(&rsi);
-        if (rdbSave(server.rdb_filename,rsiptr) != C_OK) {
-            addReply(c,shared.err);
-            return;
+        int flush = 1, save = 1;
+        int flags = RDBFLAGS_NONE;
+
+        /* Parse the additional options that modify the RELOAD
+         * behavior. */
+        for (int j = 2; j < c->argc; j++) {
+            char *opt = c->argv[j]->ptr;
+            if (!strcasecmp(opt,"MERGE")) {
+                flags |= RDBFLAGS_ALLOW_DUP;
+            } else if (!strcasecmp(opt,"NOFLUSH")) {
+                flush = 0;
+            } else if (!strcasecmp(opt,"NOSAVE")) {
+                save = 0;
+            } else {
+                addReplyError(c,"DEBUG RELOAD only supports the "
+                                "MERGE, NOFLUSH and NOSAVE options.");
+                return;
+            }
         }
-        emptyDb(-1,EMPTYDB_NO_FLAGS,NULL);
+
+        /* The default beahvior is to save the RDB file before loading
+         * it back. */
+        if (save) {
+            rdbSaveInfo rsi, *rsiptr;
+            rsiptr = rdbPopulateSaveInfo(&rsi);
+            if (rdbSave(server.rdb_filename,rsiptr) != C_OK) {
+                addReply(c,shared.err);
+                return;
+            }
+        }
+
+        /* The default behavior is to remove the current dataset from
+         * memory before loading the RDB file, however when MERGE is
+         * used together with NOFLUSH, we are able to merge two datasets. */
+        if (flush) emptyDb(-1,EMPTYDB_NO_FLAGS,NULL);
+
         protectClient(c);
-        int ret = rdbLoad(server.rdb_filename,NULL,RDBFLAGS_NONE);
+        int ret = rdbLoad(server.rdb_filename,NULL,flags);
         unprotectClient(c);
         if (ret != C_OK) {
             addReplyError(c,"Error trying to load the RDB dump");
diff --git a/src/rdb.c b/src/rdb.c
index 313830c25..3f25535ab 100644
--- a/src/rdb.c
+++ b/src/rdb.c
@@ -2242,18 +2242,28 @@ int rdbLoadRio(rio *rdb, int rdbflags, rdbSaveInfo *rsi) {
             sdsfree(key);
             decrRefCount(val);
         } else {
+            robj keyobj;
+
             /* Add the new object in the hash table */
             int retval = dictAdd(db->dict, key, val);
             if (retval != DICT_OK) {
-                serverLog(LL_WARNING,
-                    "RDB has duplicated key '%s' in DB %d",key,db->id);
-                serverPanic("Duplicated key found in RDB file");
+                if (rdbflags & RDBFLAGS_ALLOW_DUP) {
+                    /* This flag is useful for DEBUG RELOAD special modes.
+                     * When it's set we allow new keys to replace the current
+                     * keys with the same name. */
+                    initStaticStringObject(keyobj,key);
+                    dbSyncDelete(db,&keyobj);
+                    dictAdd(db->dict, key, val);
+                } else {
+                    serverLog(LL_WARNING,
+                        "RDB has duplicated key '%s' in DB %d",key,db->id);
+                    serverPanic("Duplicated key found in RDB file");
+                }
             }
             if (server.cluster_enabled) slotToKeyAdd(key);
 
             /* Set the expire time if needed */
             if (expiretime != -1) {
-                robj keyobj;
                 initStaticStringObject(keyobj,key);
                 setExpire(NULL,db,&keyobj,expiretime);
             }
diff --git a/src/rdb.h b/src/rdb.h
index 9dfcae7a5..aae682dbc 100644
--- a/src/rdb.h
+++ b/src/rdb.h
@@ -122,10 +122,10 @@
 #define RDB_LOAD_SDS    (1<<2)
 
 /* flags on the purpose of rdb save or load */
-#define RDBFLAGS_NONE 0
-#define RDBFLAGS_AOF_PREAMBLE (1<<0)
-#define RDBFLAGS_REPLICATION (1<<1)
-#define RDBFLAGS_ALLOW_DUP (1<<2)
+#define RDBFLAGS_NONE 0                 /* No special RDB loading. */
+#define RDBFLAGS_AOF_PREAMBLE (1<<0)    /* Load/save the RDB as AOF preamble. */
+#define RDBFLAGS_REPLICATION (1<<1)     /* Load/save for SYNC. */
+#define RDBFLAGS_ALLOW_DUP (1<<2)       /* Allow duplicated keys when loading.*/
 
 int rdbSaveType(rio *rdb, unsigned char type);
 int rdbLoadType(rio *rdb);

From 0f31bb5c1ace7569880a74e3d3569d95e45f8109 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Wed, 15 Apr 2020 15:59:52 +0200
Subject: [PATCH 051/186] Fix HELLO reply in Sentinel mode, see #6160.

---
 src/networking.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/networking.c b/src/networking.c
index 654fda517..3dd6bf5da 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -2460,7 +2460,7 @@ void helloCommand(client *c) {
 
     addReplyBulkCString(c,"mode");
     if (server.sentinel_mode) addReplyBulkCString(c,"sentinel");
-    if (server.cluster_enabled) addReplyBulkCString(c,"cluster");
+    else if (server.cluster_enabled) addReplyBulkCString(c,"cluster");
     else addReplyBulkCString(c,"standalone");
 
     if (!server.sentinel_mode) {

From f855db61b09fe655c4577600ab1f76eb53e90d9d Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Thu, 9 Apr 2020 16:20:41 +0200
Subject: [PATCH 052/186] incrRefCount(): abort on statically allocated object.

---
 src/object.c | 10 +++++++++-
 src/server.h |  4 +++-
 2 files changed, 12 insertions(+), 2 deletions(-)

diff --git a/src/object.c b/src/object.c
index 52d5b11f5..1bc400e85 100644
--- a/src/object.c
+++ b/src/object.c
@@ -347,7 +347,15 @@ void freeStreamObject(robj *o) {
 }
 
 void incrRefCount(robj *o) {
-    if (o->refcount != OBJ_SHARED_REFCOUNT) o->refcount++;
+    if (o->refcount < OBJ_FIRST_SPECIAL_REFCOUNT) {
+        o->refcount++;
+    } else {
+        if (o->refcount == OBJ_SHARED_REFCOUNT) {
+            /* Nothing to do: this refcount is immutable. */
+        } else if (o->refcount == OBJ_STATIC_REFCOUNT) {
+            serverPanic("You tried to retain an object allocated in the stack");
+        }
+    }
 }
 
 void decrRefCount(robj *o) {
diff --git a/src/server.h b/src/server.h
index b8c46153d..9691381c3 100644
--- a/src/server.h
+++ b/src/server.h
@@ -597,7 +597,9 @@ typedef struct RedisModuleDigest {
 #define LRU_CLOCK_MAX ((1<<LRU_BITS)-1) /* Max value of obj->lru */
 #define LRU_CLOCK_RESOLUTION 1000 /* LRU clock resolution in ms */
 
-#define OBJ_SHARED_REFCOUNT INT_MAX
+#define OBJ_SHARED_REFCOUNT INT_MAX     /* Global object never destroyed. */
+#define OBJ_STATIC_REFCOUNT (INT_MAX-1) /* Object allocated in the stack. */
+#define OBJ_FIRST_SPECIAL_REFCOUNT OBJ_STATIC_REFCOUNT
 typedef struct redisObject {
     unsigned type:4;
     unsigned encoding:4;

From e5b9eb8171417321388fff2ebaaeeed9b220c2f7 Mon Sep 17 00:00:00 2001
From: Guy Benoish <guy.benoish@redislabs.com>
Date: Sat, 11 Apr 2020 15:05:01 +0300
Subject: [PATCH 053/186] Typo in getTimeoutFromObjectOrReply's error reply

---
 src/timeout.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/timeout.c b/src/timeout.c
index bb5999418..7787a049f 100644
--- a/src/timeout.c
+++ b/src/timeout.c
@@ -166,7 +166,7 @@ int getTimeoutFromObjectOrReply(client *c, robj *object, mstime_t *timeout, int
 
     if (unit == UNIT_SECONDS) {
         if (getLongDoubleFromObjectOrReply(c,object,&ftval,
-            "timeout is not an float or out of range") != C_OK)
+            "timeout is not a float or out of range") != C_OK)
             return C_ERR;
         tval = (long long) (ftval * 1000.0);
     } else {

From 0f239e51b057f3a9fb359a4ce1f329d57fdb9c15 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Thu, 9 Apr 2020 16:21:48 +0200
Subject: [PATCH 054/186] RDB: refactor some RDB loading code into
 dbAddRDBLoad().

---
 src/db.c     | 18 ++++++++++++++++++
 src/rdb.c    |  7 +++----
 src/server.h |  1 +
 3 files changed, 22 insertions(+), 4 deletions(-)

diff --git a/src/db.c b/src/db.c
index d393a5fdd..59f0cc7a0 100644
--- a/src/db.c
+++ b/src/db.c
@@ -188,6 +188,24 @@ void dbAdd(redisDb *db, robj *key, robj *val) {
     if (server.cluster_enabled) slotToKeyAdd(key->ptr);
 }
 
+/* This is a special version of dbAdd() that is used only when loading
+ * keys from the RDB file: the key is passed as an SDS string that is
+ * retained by the function (and not freed by the caller).
+ *
+ * Moreover this function will not abort if the key is already busy, to
+ * give more control to the caller, nor will signal the key as ready
+ * since it is not useful in this context.
+ *
+ * The function returns 1 if the key was added to the database, taking
+ * ownership of the SDS string, otherwise 0 is returned, and is up to the
+ * caller to free the SDS string. */
+int dbAddRDBLoad(redisDb *db, sds key, robj *val) {
+    int retval = dictAdd(db->dict, key, val);
+    if (retval != DICT_OK) return 0;
+    if (server.cluster_enabled) slotToKeyAdd(key);
+    return 1;
+}
+
 /* Overwrite an existing key with a new value. Incrementing the reference
  * count of the new value is up to the caller.
  * This function does not modify the expire time of the existing key.
diff --git a/src/rdb.c b/src/rdb.c
index 3f25535ab..143b6c325 100644
--- a/src/rdb.c
+++ b/src/rdb.c
@@ -2245,22 +2245,21 @@ int rdbLoadRio(rio *rdb, int rdbflags, rdbSaveInfo *rsi) {
             robj keyobj;
 
             /* Add the new object in the hash table */
-            int retval = dictAdd(db->dict, key, val);
-            if (retval != DICT_OK) {
+            int added = dbAddRDBLoad(db,key,val);
+            if (!added) {
                 if (rdbflags & RDBFLAGS_ALLOW_DUP) {
                     /* This flag is useful for DEBUG RELOAD special modes.
                      * When it's set we allow new keys to replace the current
                      * keys with the same name. */
                     initStaticStringObject(keyobj,key);
                     dbSyncDelete(db,&keyobj);
-                    dictAdd(db->dict, key, val);
+                    dbAddRDBLoad(db,key,val);
                 } else {
                     serverLog(LL_WARNING,
                         "RDB has duplicated key '%s' in DB %d",key,db->id);
                     serverPanic("Duplicated key found in RDB file");
                 }
             }
-            if (server.cluster_enabled) slotToKeyAdd(key);
 
             /* Set the expire time if needed */
             if (expiretime != -1) {
diff --git a/src/server.h b/src/server.h
index 9691381c3..c268f2bee 100644
--- a/src/server.h
+++ b/src/server.h
@@ -2069,6 +2069,7 @@ int objectSetLRUOrLFU(robj *val, long long lfu_freq, long long lru_idle,
 #define LOOKUP_NONE 0
 #define LOOKUP_NOTOUCH (1<<0)
 void dbAdd(redisDb *db, robj *key, robj *val);
+int dbAddRDBLoad(redisDb *db, sds key, robj *val);
 void dbOverwrite(redisDb *db, robj *key, robj *val);
 void genericSetKey(redisDb *db, robj *key, robj *val, int keepttl, int signal);
 void setKey(redisDb *db, robj *key, robj *val);

From a7ee3c3e77d04ac98c68b43ef18231eefd1f2615 Mon Sep 17 00:00:00 2001
From: liumiuyong <lmy@meitu.com>
Date: Thu, 9 Apr 2020 17:48:29 +0800
Subject: [PATCH 055/186] FIX: truncate max/min longitude,latitude related
 geo_point (ex:  {180, 85.05112878} )

---
 src/geohash.c | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/geohash.c b/src/geohash.c
index db5ae025a..de9620b7a 100644
--- a/src/geohash.c
+++ b/src/geohash.c
@@ -206,7 +206,11 @@ int geohashDecodeWGS84(const GeoHashBits hash, GeoHashArea *area) {
 int geohashDecodeAreaToLongLat(const GeoHashArea *area, double *xy) {
     if (!xy) return 0;
     xy[0] = (area->longitude.min + area->longitude.max) / 2;
+    if (xy[0] > GEO_LONG_MAX) xy[0] = GEO_LONG_MAX;
+    if (xy[0] < GEO_LONG_MIN) xy[0] = GEO_LONG_MIN;
     xy[1] = (area->latitude.min + area->latitude.max) / 2;
+    if (xy[1] > GEO_LAT_MAX) xy[1] = GEO_LAT_MAX;
+    if (xy[1] < GEO_LAT_MIN) xy[1] = GEO_LAT_MIN;
     return 1;
 }
 

From 414debfd04d21c0b19d5f6bb5249398d83670f3d Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Wed, 8 Apr 2020 12:55:57 +0200
Subject: [PATCH 056/186] Speedup: unblock clients on keys in O(1).

See #7071.
---
 src/adlist.c  | 20 +++++++++++++++++---
 src/adlist.h  |  3 ++-
 src/blocked.c | 48 ++++++++++++++++++++++++++++++------------------
 src/server.c  |  2 +-
 4 files changed, 50 insertions(+), 23 deletions(-)

diff --git a/src/adlist.c b/src/adlist.c
index ec5f8bbf4..0fedc0729 100644
--- a/src/adlist.c
+++ b/src/adlist.c
@@ -327,12 +327,11 @@ listNode *listIndex(list *list, long index) {
 }
 
 /* Rotate the list removing the tail node and inserting it to the head. */
-void listRotate(list *list) {
-    listNode *tail = list->tail;
-
+void listRotateTailToHead(list *list) {
     if (listLength(list) <= 1) return;
 
     /* Detach current tail */
+    listNode *tail = list->tail;
     list->tail = tail->prev;
     list->tail->next = NULL;
     /* Move it as head */
@@ -342,6 +341,21 @@ void listRotate(list *list) {
     list->head = tail;
 }
 
+/* Rotate the list removing the head node and inserting it to the tail. */
+void listRotateHeadToTail(list *list) {
+    if (listLength(list) <= 1) return;
+
+    listNode *head = list->head;
+    /* Detach current head */
+    list->head = head->next;
+    list->head->prev = NULL;
+    /* Move it as tail */
+    list->tail->next = head;
+    head->next = NULL;
+    head->prev = list->tail;
+    list->tail = head;
+}
+
 /* Add all the elements of the list 'o' at the end of the
  * list 'l'. The list 'other' remains empty but otherwise valid. */
 void listJoin(list *l, list *o) {
diff --git a/src/adlist.h b/src/adlist.h
index 28b9016ce..dd8a8d693 100644
--- a/src/adlist.h
+++ b/src/adlist.h
@@ -85,7 +85,8 @@ listNode *listSearchKey(list *list, void *key);
 listNode *listIndex(list *list, long index);
 void listRewind(list *list, listIter *li);
 void listRewindTail(list *list, listIter *li);
-void listRotate(list *list);
+void listRotateTailToHead(list *list);
+void listRotateHeadToTail(list *list);
 void listJoin(list *l, list *o);
 
 /* Directions for iterators */
diff --git a/src/blocked.c b/src/blocked.c
index e3a803ae3..00cc798d5 100644
--- a/src/blocked.c
+++ b/src/blocked.c
@@ -64,6 +64,21 @@
 
 int serveClientBlockedOnList(client *receiver, robj *key, robj *dstkey, redisDb *db, robj *value, int where);
 
+/* This structure represents the blocked key information that we store
+ * in the client structure. Each client blocked on keys, has a
+ * client->bpop.keys hash table. The keys of the hash table are Redis
+ * keys pointers to 'robj' structures. The value is this structure.
+ * The structure has two goals: firstly we store the list node that this
+ * client uses to be listed in the database "blocked clients for this key"
+ * list, so we can later unblock in O(1) without a list scan.
+ * Secondly for certain blocking types, we have additional info. Right now
+ * the only use for additional info we have is when clients are blocked
+ * on streams, as we have to remember the ID it blocked for. */
+typedef struct bkinfo {
+    listNode *listnode;     /* List node for db->blocking_keys[key] list. */
+    streamID stream_id;     /* Stream ID if we blocked in a stream. */
+} bkinfo;
+
 /* Block a client for the specific operation type. Once the CLIENT_BLOCKED
  * flag is set client query buffer is not longer processed, but accumulated,
  * and will be processed when the client is unblocked. */
@@ -211,8 +226,7 @@ void serveClientsBlockedOnListKey(robj *o, readyList *rl) {
             if (receiver->btype != BLOCKED_LIST) {
                 /* Put at the tail, so that at the next call
                  * we'll not run into it again. */
-                listDelNode(clients,clientnode);
-                listAddNodeTail(clients,receiver);
+                listRotateHeadToTail(clients);
                 continue;
             }
 
@@ -273,8 +287,7 @@ void serveClientsBlockedOnSortedSetKey(robj *o, readyList *rl) {
             if (receiver->btype != BLOCKED_ZSET) {
                 /* Put at the tail, so that at the next call
                  * we'll not run into it again. */
-                listDelNode(clients,clientnode);
-                listAddNodeTail(clients,receiver);
+                listRotateHeadToTail(clients);
                 continue;
             }
 
@@ -320,8 +333,8 @@ void serveClientsBlockedOnStreamKey(robj *o, readyList *rl) {
         while((ln = listNext(&li))) {
             client *receiver = listNodeValue(ln);
             if (receiver->btype != BLOCKED_STREAM) continue;
-            streamID *gt = dictFetchValue(receiver->bpop.keys,
-                                          rl->key);
+            bkinfo *bki = dictFetchValue(receiver->bpop.keys,rl->key);
+            streamID *gt = &bki->stream_id;
 
             /* If we blocked in the context of a consumer
              * group, we need to resolve the group and update the
@@ -419,8 +432,7 @@ void serveClientsBlockedOnKeyByModule(readyList *rl) {
              * ready to be served, so they'll remain in the list
              * sometimes. We want also be able to skip clients that are
              * not blocked for the MODULE type safely. */
-            listDelNode(clients,clientnode);
-            listAddNodeTail(clients,receiver);
+            listRotateHeadToTail(clients);
 
             if (receiver->btype != BLOCKED_MODULE) continue;
 
@@ -551,17 +563,15 @@ void blockForKeys(client *c, int btype, robj **keys, int numkeys, mstime_t timeo
     if (target != NULL) incrRefCount(target);
 
     for (j = 0; j < numkeys; j++) {
-        /* The value associated with the key name in the bpop.keys dictionary
-         * is NULL for lists and sorted sets, or the stream ID for streams. */
-        void *key_data = NULL;
-        if (btype == BLOCKED_STREAM) {
-            key_data = zmalloc(sizeof(streamID));
-            memcpy(key_data,ids+j,sizeof(streamID));
-        }
+        /* Allocate our bkinfo structure, associated to each key the client
+         * is blocked for. */
+        bkinfo *bki = zmalloc(sizeof(*bki));
+        if (btype == BLOCKED_STREAM)
+            bki->stream_id = ids[j];
 
         /* If the key already exists in the dictionary ignore it. */
-        if (dictAdd(c->bpop.keys,keys[j],key_data) != DICT_OK) {
-            zfree(key_data);
+        if (dictAdd(c->bpop.keys,keys[j],bki) != DICT_OK) {
+            zfree(bki);
             continue;
         }
         incrRefCount(keys[j]);
@@ -580,6 +590,7 @@ void blockForKeys(client *c, int btype, robj **keys, int numkeys, mstime_t timeo
             l = dictGetVal(de);
         }
         listAddNodeTail(l,c);
+        bki->listnode = listLast(l);
     }
     blockClient(c,btype);
 }
@@ -596,11 +607,12 @@ void unblockClientWaitingData(client *c) {
     /* The client may wait for multiple keys, so unblock it for every key. */
     while((de = dictNext(di)) != NULL) {
         robj *key = dictGetKey(de);
+        bkinfo *bki = dictGetVal(de);
 
         /* Remove this client from the list of clients waiting for this key. */
         l = dictFetchValue(c->db->blocking_keys,key);
         serverAssertWithInfo(c,key,l != NULL);
-        listDelNode(l,listSearchKey(l,c));
+        listDelNode(l,bki->listnode);
         /* If the list is empty we need to remove it to avoid wasting memory */
         if (listLength(l) == 0)
             dictDelete(c->db->blocking_keys,key);
diff --git a/src/server.c b/src/server.c
index 996e0f5d2..0afd67514 100644
--- a/src/server.c
+++ b/src/server.c
@@ -1666,7 +1666,7 @@ void clientsCron(void) {
         /* Rotate the list, take the current head, process.
          * This way if the client must be removed from the list it's the
          * first element and we don't incur into O(N) computation. */
-        listRotate(server.clients);
+        listRotateTailToHead(server.clients);
         head = listFirst(server.clients);
         c = listNodeValue(head);
         /* The following functions do different service checks on the client.

From bec68bff2978de88b23140f41b6a943162333b11 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Thu, 9 Apr 2020 16:25:30 +0200
Subject: [PATCH 057/186] Use the special static refcount for stack objects.

---
 src/server.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/server.h b/src/server.h
index c268f2bee..d0d5ff154 100644
--- a/src/server.h
+++ b/src/server.h
@@ -620,7 +620,7 @@ char *getObjectTypeName(robj*);
  * we'll update it when the structure is changed, to avoid bugs like
  * bug #85 introduced exactly in this way. */
 #define initStaticStringObject(_var,_ptr) do { \
-    _var.refcount = 1; \
+    _var.refcount = OBJ_STATIC_REFCOUNT; \
     _var.type = OBJ_STRING; \
     _var.encoding = OBJ_ENCODING_RAW; \
     _var.ptr = _ptr; \

From b86140ac5fb30b0dd4e36ca71b27c2ca1275d8da Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Wed, 15 Apr 2020 16:12:06 +0200
Subject: [PATCH 058/186] Don't allow empty spaces in ACL key patterns.

Fixes issue #6418.
---
 src/acl.c | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/src/acl.c b/src/acl.c
index 6847130ad..a5e35c4d1 100644
--- a/src/acl.c
+++ b/src/acl.c
@@ -30,6 +30,7 @@
 #include "server.h"
 #include "sha256.h"
 #include <fcntl.h>
+#include <ctype.h>
 
 /* =============================================================================
  * Global state for ACLs
@@ -690,7 +691,8 @@ void ACLAddAllowedSubcommand(user *u, unsigned long id, const char *sub) {
  *
  * When an error is returned, errno is set to the following values:
  *
- * EINVAL: The specified opcode is not understood.
+ * EINVAL: The specified opcode is not understood or the key pattern is
+ *         invalid (contains non allowed characters).
  * ENOENT: The command name or command category provided with + or - is not
  *         known.
  * EBUSY:  The subcommand you want to add is about a command that is currently
@@ -789,6 +791,15 @@ int ACLSetUser(user *u, const char *op, ssize_t oplen) {
             errno = EEXIST;
             return C_ERR;
         }
+        /* Validate the pattern: no spaces nor null characters
+         * are allowed, for simpler rewriting of the ACLs without
+         * using quoting. */
+        for (int i = 1; i < oplen; i++) {
+            if (isspace(op[i]) || op[i] == 0) {
+                errno = EINVAL;
+                return C_ERR;
+            }
+        }
         sds newpat = sdsnewlen(op+1,oplen-1);
         listNode *ln = listSearchKey(u->patterns,newpat);
         /* Avoid re-adding the same pattern multiple times. */

From 371ab0cff88a0fa85253d09db32716dafe98b5b4 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Wed, 15 Apr 2020 16:39:42 +0200
Subject: [PATCH 059/186] Don't allow empty spaces in ACL usernames.

Fixes issue #6418.
---
 src/acl.c | 44 ++++++++++++++++++++++++++++++++++++--------
 1 file changed, 36 insertions(+), 8 deletions(-)

diff --git a/src/acl.c b/src/acl.c
index a5e35c4d1..75b954c5e 100644
--- a/src/acl.c
+++ b/src/acl.c
@@ -170,6 +170,18 @@ sds ACLHashPassword(unsigned char *cleartext, size_t len) {
  * Low level ACL API
  * ==========================================================================*/
 
+/* Return 1 if the specified string contains spaces or null characters.
+ * We do this for usernames and key patterns for simpler rewriting of
+ * ACL rules, presentation on ACL list, and to avoid subtle security bugs
+ * that may arise from parsing the rules in presence of escapes.
+ * The function returns 0 if the string has no spaces. */
+int ACLStringHasSpaces(const char *s, size_t len) {
+    for (size_t i = 0; i < len; i++) {
+        if (isspace(s[i]) || s[i] == 0) return 1;
+    }
+    return 0;
+}
+
 /* Given the category name the command returns the corresponding flag, or
  * zero if there is no match. */
 uint64_t ACLGetCommandCategoryFlagByName(const char *name) {
@@ -791,14 +803,9 @@ int ACLSetUser(user *u, const char *op, ssize_t oplen) {
             errno = EEXIST;
             return C_ERR;
         }
-        /* Validate the pattern: no spaces nor null characters
-         * are allowed, for simpler rewriting of the ACLs without
-         * using quoting. */
-        for (int i = 1; i < oplen; i++) {
-            if (isspace(op[i]) || op[i] == 0) {
-                errno = EINVAL;
-                return C_ERR;
-            }
+        if (ACLStringHasSpaces(op+1,oplen-1)) {
+            errno = EINVAL;
+            return C_ERR;
         }
         sds newpat = sdsnewlen(op+1,oplen-1);
         listNode *ln = listSearchKey(u->patterns,newpat);
@@ -1175,6 +1182,12 @@ int ACLLoadConfiguredUsers(void) {
     while ((ln = listNext(&li)) != NULL) {
         sds *aclrules = listNodeValue(ln);
         sds username = aclrules[0];
+
+        if (ACLStringHasSpaces(aclrules[0],sdslen(aclrules[0]))) {
+            serverLog(LL_WARNING,"Spaces not allowed in ACL usernames");
+            return C_ERR;
+        }
+
         user *u = ACLCreateUser(username,sdslen(username));
         if (!u) {
             u = ACLGetUserByName(username,sdslen(username));
@@ -1300,6 +1313,14 @@ sds ACLLoadFromFile(const char *filename) {
             continue;
         }
 
+        /* Spaces are not allowed in usernames. */
+        if (ACLStringHasSpaces(argv[1],sdslen(argv[1]))) {
+            errors = sdscatprintf(errors,
+                     "'%s:%d: username '%s' contains invalid characters. ",
+                     server.acl_filename, linenum, argv[1]);
+            continue;
+        }
+
         /* Try to process the line using the fake user to validate iif
          * the rules are able to apply cleanly. */
         ACLSetUser(fakeuser,"reset",-1);
@@ -1609,6 +1630,13 @@ void aclCommand(client *c) {
     char *sub = c->argv[1]->ptr;
     if (!strcasecmp(sub,"setuser") && c->argc >= 3) {
         sds username = c->argv[2]->ptr;
+        /* Check username validity. */
+        if (ACLStringHasSpaces(username,sdslen(username))) {
+            addReplyErrorFormat(c,
+                "Usernames can't contain spaces or null characters");
+            return;
+        }
+
         /* Create a temporary user to validate and stage all changes against
          * before applying to an existing user or creating a new user. If all
          * arguments are valid the user parameters will all be applied together.

From 48781dd95cdd1d35c8436c91f0414ca26721a34d Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Thu, 16 Apr 2020 11:21:52 +0200
Subject: [PATCH 060/186] RESP3: fix HELLO map len in Sentinel mode.

See #6160.
---
 src/networking.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/networking.c b/src/networking.c
index 3dd6bf5da..8f3d79170 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -2444,7 +2444,7 @@ void helloCommand(client *c) {
 
     /* Let's switch to the specified RESP mode. */
     c->resp = ver;
-    addReplyMapLen(c,7);
+    addReplyMapLen(c,6 + !server.sentinel_mode);
 
     addReplyBulkCString(c,"server");
     addReplyBulkCString(c,"redis");

From 9f594e243288a45045593c2db3b470212ac05269 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Thu, 16 Apr 2020 16:08:37 +0200
Subject: [PATCH 061/186] Update SDS to latest version.

---
 src/sds.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/sds.c b/src/sds.c
index 98bd2e77f..118971621 100644
--- a/src/sds.c
+++ b/src/sds.c
@@ -97,11 +97,11 @@ sds sdsnewlen(const void *init, size_t initlen) {
     unsigned char *fp; /* flags pointer. */
 
     sh = s_malloc(hdrlen+initlen+1);
+    if (sh == NULL) return NULL;
     if (init==SDS_NOINIT)
         init = NULL;
     else if (!init)
         memset(sh, 0, hdrlen+initlen+1);
-    if (sh == NULL) return NULL;
     s = (char*)sh+hdrlen;
     fp = ((unsigned char*)s)-1;
     switch(type) {

From 7cf0a77d59840fe3b1cdc5a98c91ce99c61fd3e3 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Thu, 16 Apr 2020 16:18:02 +0200
Subject: [PATCH 062/186] Redis 6.0-RC4.

---
 00-RELEASENOTES | 269 ++++++++++++++++++++++++++++++++++++++++++++++++
 src/version.h   |   2 +-
 2 files changed, 270 insertions(+), 1 deletion(-)

diff --git a/00-RELEASENOTES b/00-RELEASENOTES
index 290158efb..223e6e66c 100644
--- a/00-RELEASENOTES
+++ b/00-RELEASENOTES
@@ -11,6 +11,275 @@ CRITICAL: There is a critical bug affecting MOST USERS. Upgrade ASAP.
 SECURITY: There are security fixes in the release.
 --------------------------------------------------------------------------------
 
+================================================================================
+Redis 6.0-rc4     Released Thu Apr 16 16:10:35 CEST 2020
+================================================================================
+
+Upgrade urgency LOW: If you are using RC3 without issues, don't rush.
+
+Hi all, this the latest release candidate of Redis 6. This is likely to
+be very similar to what you'll see in Redis 6 GA. Please test it and
+report any issue :-)
+
+Main changes in this release:
+
+    * Big INFO speedup when using a lot of of clients.
+    * Big speedup on all the blocking commands: now blocking
+      on the same key is O(1) instead of being O(N).
+    * Stale replicas now allow MULTI/EXEC.
+    * New command: LCS (Longest Common Subsequence).
+    * Add a new configuration to make DEL like UNLINK.
+    * RDB loading speedup.
+    * Many bugs fixed (see the commit messages at the end of this node)
+
+See you in 14 days for Redis 6 GA.
+
+List of commits:
+
+antirez in commit 9f594e243:
+ Update SDS to latest version.
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+antirez in commit 48781dd95:
+ RESP3: fix HELLO map len in Sentinel mode.
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+antirez in commit 371ab0cff:
+ Don't allow empty spaces in ACL usernames.
+ 1 file changed, 36 insertions(+), 8 deletions(-)
+
+antirez in commit b86140ac5:
+ Don't allow empty spaces in ACL key patterns.
+ 1 file changed, 12 insertions(+), 1 deletion(-)
+
+liumiuyong in commit a7ee3c3e7:
+ FIX: truncate max/min longitude,latitude related geo_point (ex:  {180, 85.05112878} )
+ 1 file changed, 4 insertions(+)
+
+Guy Benoish in commit e5b9eb817:
+ Typo in getTimeoutFromObjectOrReply's error reply
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+antirez in commit 0f31bb5c1:
+ Fix HELLO reply in Sentinel mode, see #6160.
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+hwware in commit b92d9a895:
+ fix spelling in acl.c
+ 1 file changed, 2 insertions(+), 2 deletions(-)
+
+antirez in commit 8f896e57a:
+ Fix zsetAdd() top comment spelling.
+ 1 file changed, 3 insertions(+), 3 deletions(-)
+
+hayleeliu in commit 8f5157058:
+ fix spelling mistake in bitops.c
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+antirez in commit ddeda9ceb:
+ Fix function names in zslDeleteNode() top comment.
+ 1 file changed, 2 insertions(+), 1 deletion(-)
+
+antirez in commit bde1f0a8e:
+ RESP3: change streams items from maps to arrays.
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+antirez in commit bec68bff2:
+ Use the special static refcount for stack objects.
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+antirez in commit 0f239e51b:
+ RDB: refactor some RDB loading code into dbAddRDBLoad().
+ 3 files changed, 22 insertions(+), 4 deletions(-)
+
+antirez in commit f855db61b:
+ incrRefCount(): abort on statically allocated object.
+ 2 files changed, 12 insertions(+), 2 deletions(-)
+
+antirez in commit 23094ba01:
+ More powerful DEBUG RELOAD.
+ 3 files changed, 55 insertions(+), 16 deletions(-)
+
+antirez in commit 8161a7a3e:
+ RDB: clarify a condition in rdbLoadRio().
+ 2 files changed, 9 insertions(+), 2 deletions(-)
+
+antirez in commit 61b153073:
+ RDB: load files faster avoiding useless free+realloc.
+ 7 files changed, 40 insertions(+), 28 deletions(-)
+
+antirez in commit 414debfd0:
+ Speedup: unblock clients on keys in O(1).
+ 4 files changed, 50 insertions(+), 23 deletions(-)
+
+antirez in commit cbcd07777:
+ Fix ACL HELP table missing comma.
+ 1 file changed, 12 insertions(+), 12 deletions(-)
+
+mymilkbottles in commit 2437455f2:
+ Judge the log level in advance
+ 1 file changed, 1 insertion(+)
+
+antirez in commit 35c64b898:
+ Speedup INFO by counting client memory incrementally.
+ 4 files changed, 52 insertions(+), 26 deletions(-)
+
+qetu3790 in commit c3ac71748:
+ fix comments about RESIZE DB opcode in rdb.c
+ 1 file changed, 1 insertion(+), 4 deletions(-)
+
+antirez in commit c8dbcff9d:
+ Clarify redis.conf comment about lazyfree-lazy-user-del.
+ 1 file changed, 9 insertions(+), 5 deletions(-)
+
+zhaozhao.zz in commit abd5156f2:
+ lazyfree: add a new configuration lazyfree-lazy-user-del
+ 4 files changed, 7 insertions(+), 2 deletions(-)
+
+antirez in commit 5719b3054:
+ LCS: more tests.
+ 1 file changed, 8 insertions(+)
+
+antirez in commit c89e1f293:
+ LCS: allow KEYS / STRINGS to be anywhere.
+ 1 file changed, 6 deletions(-)
+
+antirez in commit 0b16f8d44:
+ LCS tests.
+ 1 file changed, 22 insertions(+)
+
+antirez in commit 9254a805d:
+ LCS: get rid of STOREIDX option. Fix get keys helper.
+ 2 files changed, 20 insertions(+), 21 deletions(-)
+
+antirez in commit a4c490703:
+ LCS: fix stale comment.
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+antirez in commit cb92c23de:
+ LCS: output LCS len as well in IDX mode.
+ 1 file changed, 6 insertions(+), 1 deletion(-)
+
+antirez in commit 56a52e804:
+ LCS: MINMATCHLEN and WITHMATCHLEN options.
+ 1 file changed, 24 insertions(+), 11 deletions(-)
+
+antirez in commit ebb09a5c3:
+ LCS: 7x speedup by accessing the array with better locality.
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+antirez in commit a9f8a8cba:
+ LCS: implement KEYS option.
+ 1 file changed, 18 insertions(+), 2 deletions(-)
+
+antirez in commit 4aa24e62a:
+ LCS: other fixes to range emission.
+ 1 file changed, 20 insertions(+), 16 deletions(-)
+
+antirez in commit 2b67b6b87:
+ LCS: fix emission of last range starting at index 0.
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+antirez in commit 420aac727:
+ LCS: implement range indexes option.
+ 1 file changed, 59 insertions(+), 9 deletions(-)
+
+antirez in commit a518a9a76:
+ LCS: initial functionality implemented.
+ 4 files changed, 156 insertions(+), 1 deletion(-)
+
+srzhao in commit 026cc11b0:
+ Check OOM at script start to get stable lua OOM state.
+ 3 files changed, 11 insertions(+), 4 deletions(-)
+
+Oran Agra in commit 02b594f6a:
+ diffrent fix for runtest --host --port
+ 2 files changed, 13 insertions(+), 13 deletions(-)
+
+Guy Benoish in commit f695d1830:
+ Try to fix time-sensitive tests in blockonkey.tcl
+ 1 file changed, 54 insertions(+), 1 deletion(-)
+
+Guy Benoish in commit 0e42cfc36:
+ Use __attribute__ only if __GNUC__ is defined
+ 1 file changed, 12 insertions(+), 3 deletions(-)
+
+Guy Benoish in commit 91ed9b3c4:
+ Modules: Perform printf-like format checks in variadic API
+ 1 file changed, 3 insertions(+), 3 deletions(-)
+
+Valentino Geron in commit 3e0d20962:
+ XREAD and XREADGROUP should not be allowed from scripts when BLOCK option is being used
+ 3 files changed, 18 insertions(+), 2 deletions(-)
+
+Guy Benoish in commit 240094c9b:
+ Stale replica should allow MULTI/EXEC
+ 1 file changed, 3 insertions(+), 3 deletions(-)
+
+Xudong Zhang in commit 209f3a1eb:
+ fix integer overflow
+ 1 file changed, 2 insertions(+), 2 deletions(-)
+
+Guy Benoish in commit 024c380b9:
+ Fix no-negative-zero test
+ 1 file changed, 1 insertion(+)
+
+Oran Agra in commit a38ff404b:
+ modules don't signalModifiedKey in setKey() since that's done (optionally) in RM_CloseKey
+ 4 files changed, 8 insertions(+), 8 deletions(-)
+
+Oran Agra in commit 814874d68:
+ change CI to build and run the module api tests
+ 1 file changed, 2 insertions(+)
+
+Oran Agra in commit 061616c1b:
+ fix possible warning on incomplete struct init
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+Guy Benoish in commit 7764996be:
+ Make sure Redis does not reply with negative zero
+ 2 files changed, 10 insertions(+)
+
+Guy Benoish in commit eba28e2ce:
+ DEBUG OBJECT should pass keyname to module when loading
+ 3 files changed, 4 insertions(+), 4 deletions(-)
+
+David Carlier in commit 15c9e79a7:
+ debug, dump registers on arm too.
+ 1 file changed, 55 insertions(+), 27 deletions(-)
+
+hwware in commit cd2b5df97:
+ fix spelling in cluster.c
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+Valentino Geron in commit 8cdc153f5:
+ XACK should be executed in a "all or nothing" fashion.
+ 2 files changed, 23 insertions(+), 1 deletion(-)
+
+hwware in commit b35407fa7:
+ add check for not switching between optin optout mode directly
+ 1 file changed, 12 insertions(+), 1 deletion(-)
+
+hwware in commit 4395889c9:
+ add check for not providing both optin optout flag
+ 1 file changed, 8 insertions(+)
+
+Guy Benoish in commit 1907e0f18:
+ PERSIST should notify a keyspace event
+ 1 file changed, 1 insertion(+)
+
+Guy Benoish in commit c35a53169:
+ streamReplyWithRange: Redundant XSETIDs to replica
+ 1 file changed, 2 insertions(+), 1 deletion(-)
+
+antirez in commit 6fe66e096:
+ Simplify comment in moduleTryServeClientBlockedOnKey().
+ 1 file changed, 3 insertions(+), 12 deletions(-)
+
+Guy Benoish in commit 193fc241c:
+ Fix memory corruption in moduleHandleBlockedClients
+ 3 files changed, 149 insertions(+), 46 deletions(-)
+
 ================================================================================
 Redis 6.0-rc3     Released Tue Mar 31 17:42:39 CEST 2020
 ================================================================================
diff --git a/src/version.h b/src/version.h
index 45465341c..c240a174e 100644
--- a/src/version.h
+++ b/src/version.h
@@ -1 +1 @@
-#define REDIS_VERSION "5.9.103"
+#define REDIS_VERSION "5.9.104"

From c39f16c42273b52a2f14701bdc926f3767e8f3fc Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Fri, 17 Apr 2020 12:38:12 +0200
Subject: [PATCH 063/186] Fix XCLAIM propagation in AOF/replicas for blocking
 XREADGROUP.

See issue #7105.
---
 src/server.c   | 9 +++++++--
 src/t_stream.c | 2 +-
 2 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/src/server.c b/src/server.c
index 0afd67514..fc9b87aae 100644
--- a/src/server.c
+++ b/src/server.c
@@ -3082,8 +3082,13 @@ struct redisCommand *lookupCommandOrOriginal(sds name) {
  * + PROPAGATE_AOF (propagate into the AOF file if is enabled)
  * + PROPAGATE_REPL (propagate into the replication link)
  *
- * This should not be used inside commands implementation. Use instead
- * alsoPropagate(), preventCommandPropagation(), forceCommandPropagation().
+ * This should not be used inside commands implementation since it will not
+ * wrap the resulting commands in MULTI/EXEC. Use instead alsoPropagate(),
+ * preventCommandPropagation(), forceCommandPropagation().
+ *
+ * However for functions that need to (also) propagate out of the context of a
+ * command execution, for example when serving a blocked client, you
+ * want to use propagate().
  */
 void propagate(struct redisCommand *cmd, int dbid, robj **argv, int argc,
                int flags)
diff --git a/src/t_stream.c b/src/t_stream.c
index 4ce3a9b25..155167af9 100644
--- a/src/t_stream.c
+++ b/src/t_stream.c
@@ -848,7 +848,7 @@ void streamPropagateXCLAIM(client *c, robj *key, streamCG *group, robj *groupnam
     argv[11] = createStringObject("JUSTID",6);
     argv[12] = createStringObject("LASTID",6);
     argv[13] = createObjectFromStreamID(&group->last_id);
-    alsoPropagate(server.xclaimCommand,c->db->id,argv,14,PROPAGATE_AOF|PROPAGATE_REPL);
+    propagate(server.xclaimCommand,c->db->id,argv,14,PROPAGATE_AOF|PROPAGATE_REPL);
     decrRefCount(argv[0]);
     decrRefCount(argv[3]);
     decrRefCount(argv[4]);

From 51d3012d486113064c2f348ca96f286ff9a72522 Mon Sep 17 00:00:00 2001
From: Jamie Scott <jamie@redislabs.com>
Date: Sun, 12 Apr 2020 00:10:19 -0700
Subject: [PATCH 064/186] Adding acllog-max-len to Redis.conf

While playing with ACLs I noticed that acllog-max-len wasn't in the redis.conf, but was a supported config.

This PR documents and adds the directive to the redis.conf file.
---
 redis.conf | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/redis.conf b/redis.conf
index 5baeae65f..fab0a5898 100644
--- a/redis.conf
+++ b/redis.conf
@@ -737,6 +737,15 @@ replica-priority 100
 # For more information about ACL configuration please refer to
 # the Redis web site at https://redis.io/topics/acl
 
+# ACL LOG
+#
+# The ACL Log tracks failed commands and authentication events associated
+# with ACLs. The ACL Log is useful to troubleshoot failed commands blocked 
+# by ACLs. The ACL Log is stored in and consumes memory. There is no limit
+# to its length.You can reclaim memory with ACL LOG RESET or set a maximum
+# length below.
+acllog-max-len 128
+
 # Using an external ACL file
 #
 # Instead of configuring users here in this file, it is possible to use

From 6148f949392aa060db5d4de68966dfbc03cda212 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Thu, 16 Apr 2020 11:05:03 +0300
Subject: [PATCH 065/186] testsuite run the defrag latency test solo

this test is time sensitive and it sometimes fail to pass below the
latency threshold, even on strong machines.

this test was the reson we're running just 2 parallel tests in the
github actions CI, revering this.
---
 .github/workflows/ci.yml     |  4 ++--
 tests/test_helper.tcl        | 38 ++++++++++++++++++++++++++++++++++++
 tests/unit/memefficiency.tcl |  2 ++
 3 files changed, 42 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 3a81d1a08..551fb2d91 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -12,9 +12,9 @@ jobs:
     - name: test
       run: |
         sudo apt-get install tcl8.5
-        ./runtest --clients 2 --verbose
+        ./runtest --verbose
     - name: module api test
-      run: ./runtest-moduleapi --clients 2 --verbose
+      run: ./runtest-moduleapi --verbose
 
   build-ubuntu-old:
     runs-on: ubuntu-16.04
diff --git a/tests/test_helper.tcl b/tests/test_helper.tcl
index d80cb6907..11a804bdc 100644
--- a/tests/test_helper.tcl
+++ b/tests/test_helper.tcl
@@ -82,6 +82,7 @@ set ::skiptests {}
 set ::allowtags {}
 set ::only_tests {}
 set ::single_tests {}
+set ::run_solo_tests {}
 set ::skip_till ""
 set ::external 0; # If "1" this means, we are running against external instance
 set ::file ""; # If set, runs only the tests in this comma separated list
@@ -112,6 +113,11 @@ proc execute_tests name {
     send_data_packet $::test_server_fd done "$name"
 }
 
+proc execute_code {name code} {
+    eval $code
+    send_data_packet $::test_server_fd done "$name"
+}
+
 # Setup a list to hold a stack of server configs. When calls to start_server
 # are nested, use "srv 0 pid" to get the pid of the inner server. To access
 # outer servers, use "srv -1 pid" etcetera.
@@ -188,6 +194,18 @@ proc s {args} {
     status [srv $level "client"] [lindex $args 0]
 }
 
+# Test wrapped into run_solo are sent back from the client to the
+# test server, so that the test server will send them again to
+# clients once the clients are idle.
+proc run_solo {name code} {
+    if {$::numclients == 1 || $::loop || $::external} {
+        # run_solo is not supported in these scenarios, just run the code.
+        eval $code
+        return
+    }
+    send_data_packet $::test_server_fd run_solo [list $name $code]
+}
+
 proc cleanup {} {
     if {$::dont_clean} {
         return
@@ -337,6 +355,8 @@ proc read_from_test_client fd {
     } elseif {$status eq {server-killed}} {
         set ::active_servers [lsearch -all -inline -not -exact $::active_servers $data]
         set ::active_clients_task($fd) "(KILLED SERVER) pid:$data"
+    } elseif {$status eq {run_solo}} {
+        lappend ::run_solo_tests $data
     } else {
         if {!$::quiet} {
             puts "\[$status\]: $data"
@@ -369,6 +389,13 @@ proc force_kill_all_servers {} {
     }
 }
 
+proc lpop {listVar {count 1}} {
+    upvar 1 $listVar l
+    set ele [lindex $l 0]
+    set l [lrange $l 1 end]
+    set ele
+}
+
 # A new client is idle. Remove it from the list of active clients and
 # if there are still test units to run, launch them.
 proc signal_idle_client fd {
@@ -389,6 +416,14 @@ proc signal_idle_client fd {
         if {$::loop && $::next_test == [llength $::all_tests]} {
             set ::next_test 0
         }
+    } elseif {[llength $::run_solo_tests] != 0 && [llength $::active_clients] == 0} {
+        if {!$::quiet} {
+            puts [colorstr bold-white "Testing solo test"]
+            set ::active_clients_task($fd) "ASSIGNED: $fd solo test"
+        }
+        set ::clients_start_time($fd) [clock seconds]
+        send_data_packet $fd run_code [lpop ::run_solo_tests]
+        lappend ::active_clients $fd
     } else {
         lappend ::idle_clients $fd
         set ::active_clients_task($fd) "SLEEPING, no more units to assign"
@@ -433,6 +468,9 @@ proc test_client_main server_port {
         foreach {cmd data} $payload break
         if {$cmd eq {run}} {
             execute_tests $data
+        } elseif {$cmd eq {run_code}} {
+            foreach {name code} $data break
+            execute_code $name $code
         } else {
             error "Unknown test client command: $cmd"
         }
diff --git a/tests/unit/memefficiency.tcl b/tests/unit/memefficiency.tcl
index 06b0e07d7..777693fdf 100644
--- a/tests/unit/memefficiency.tcl
+++ b/tests/unit/memefficiency.tcl
@@ -36,6 +36,7 @@ start_server {tags {"memefficiency"}} {
     }
 }
 
+run_solo {defrag} {
 start_server {tags {"defrag"}} {
     if {[string match {*jemalloc*} [s mem_allocator]]} {
         test "Active defrag" {
@@ -328,3 +329,4 @@ start_server {tags {"defrag"}} {
         } {1}
     }
 }
+} ;# run_solo

From 5c4c73e2c2d7bc0face8271f6fa3c4f288e9a891 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Fri, 17 Apr 2020 10:51:12 +0200
Subject: [PATCH 066/186] A few comments and name changes for #7103.

---
 tests/test_helper.tcl | 17 +++++++++++++----
 1 file changed, 13 insertions(+), 4 deletions(-)

diff --git a/tests/test_helper.tcl b/tests/test_helper.tcl
index 11a804bdc..3eee1aeb7 100644
--- a/tests/test_helper.tcl
+++ b/tests/test_helper.tcl
@@ -106,14 +106,23 @@ set ::tlsdir "tests/tls"
 set ::client 0
 set ::numclients 16
 
-proc execute_tests name {
+# This function is called by one of the test clients when it receives
+# a "run" command from the server, with a filename as data.
+# It will run the specified test source file and signal it to the
+# test server when finished.
+proc execute_test_file name {
     set path "tests/$name.tcl"
     set ::curfile $path
     source $path
     send_data_packet $::test_server_fd done "$name"
 }
 
-proc execute_code {name code} {
+# This function is called by one of the test clients when it receives
+# a "run_code" command from the server, with a verbatim test source code
+# as argument, and an associated name.
+# It will run the specified code and signal it to the test server when
+# finished.
+proc execute_test_code {name code} {
     eval $code
     send_data_packet $::test_server_fd done "$name"
 }
@@ -467,10 +476,10 @@ proc test_client_main server_port {
         set payload [read $::test_server_fd $bytes]
         foreach {cmd data} $payload break
         if {$cmd eq {run}} {
-            execute_tests $data
+            execute_test_file $data
         } elseif {$cmd eq {run_code}} {
             foreach {name code} $data break
-            execute_code $name $code
+            execute_test_code $name $code
         } else {
             error "Unknown test client command: $cmd"
         }

From 3a27064c4112d751213694afec90d988a8cfd5ee Mon Sep 17 00:00:00 2001
From: omg-by <504094596@qq.com>
Date: Sat, 18 Apr 2020 00:49:16 +0800
Subject: [PATCH 067/186] fix(sentinel): sentinel.running_scripts will always
 increase more times and not reset

when trigger a always fail scripts, sentinel.running_scripts will increase ten times, however it
only decrease one times onretry the maximum. and it will't reset, when it become
SENTINEL_SCRIPT_MAX_RUNNING, sentinel don't trigger scripts.
---
 src/sentinel.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/sentinel.c b/src/sentinel.c
index d091bf230..204a8b317 100644
--- a/src/sentinel.c
+++ b/src/sentinel.c
@@ -860,6 +860,7 @@ void sentinelCollectTerminatedScripts(void) {
             sj->pid = 0;
             sj->start_time = mstime() +
                              sentinelScriptRetryDelay(sj->retry_num);
+            sentinel.running_scripts--;
         } else {
             /* Otherwise let's remove the script, but log the event if the
              * execution did not terminated in the best of the ways. */

From a76c67578ccefd233b3f4d0cb02f1810b5830e5d Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Mon, 20 Apr 2020 11:52:29 +0200
Subject: [PATCH 068/186] Sentinel: small refactoring of
 sentinelCollectTerminatedScripts().

Related to #7113.
---
 src/sentinel.c | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/sentinel.c b/src/sentinel.c
index 204a8b317..fb504ae4d 100644
--- a/src/sentinel.c
+++ b/src/sentinel.c
@@ -860,7 +860,6 @@ void sentinelCollectTerminatedScripts(void) {
             sj->pid = 0;
             sj->start_time = mstime() +
                              sentinelScriptRetryDelay(sj->retry_num);
-            sentinel.running_scripts--;
         } else {
             /* Otherwise let's remove the script, but log the event if the
              * execution did not terminated in the best of the ways. */
@@ -870,8 +869,8 @@ void sentinelCollectTerminatedScripts(void) {
             }
             listDelNode(sentinel.scripts_queue,ln);
             sentinelReleaseScriptJob(sj);
-            sentinel.running_scripts--;
         }
+        sentinel.running_scripts--;
     }
 }
 

From 3575b8706dff2b846f376153c884872bed686f0c Mon Sep 17 00:00:00 2001
From: zhenwei pi <pizhenwei@bytedance.com>
Date: Mon, 13 Apr 2020 09:58:35 +0800
Subject: [PATCH 069/186] Threaded IO: set thread name for redis-server

Set thread name for each thread of redis-server, this helps us to
monitor the utilization and optimise the performance.

And suggested-by Salvatore, implement this feature for multi
platforms. Currently support linux and bsd, ignore other OS.

An exmaple on Linux:
 # top -d 5 -p `pidof redis-server ` -H

    PID USER      PR  NI    VIRT    RES    SHR S %CPU %MEM     TIME+ COMMAND
3682671 root      20   0  227744   8248   3836 R 99.2  0.0   0:19.53 redis-server
3682677 root      20   0  227744   8248   3836 S 26.4  0.0   0:04.15 io_thd_3
3682675 root      20   0  227744   8248   3836 S 23.6  0.0   0:03.98 io_thd_1
3682676 root      20   0  227744   8248   3836 S 23.6  0.0   0:03.97 io_thd_2
3682672 root      20   0  227744   8248   3836 S  0.2  0.0   0:00.02 bio_close_file
3682673 root      20   0  227744   8248   3836 S  0.2  0.0   0:00.02 bio_aof_fsync
3682674 root      20   0  227744   8248   3836 S  0.0  0.0   0:00.00 bio_lazy_free
3682678 root      20   0  227744   8248   3836 S  0.0  0.0   0:00.00 jemalloc_bg_thd
3682682 root      20   0  227744   8248   3836 S  0.0  0.0   0:00.00 jemalloc_bg_thd
3682683 root      20   0  227744   8248   3836 S  0.0  0.0   0:00.00 jemalloc_bg_thd
3682684 root      20   0  227744   8248   3836 S  0.0  0.0   0:00.00 jemalloc_bg_thd
3682685 root      20   0  227744   8248   3836 S  0.0  0.0   0:00.00 jemalloc_bg_thd
3682687 root      20   0  227744   8248   3836 S  0.0  0.0   0:00.00 jemalloc_bg_thd

Another exmaple on FreeBSD-12.1:
  PID USERNAME    PRI NICE   SIZE    RES STATE    C   TIME    WCPU COMMAND
 5212 root        100    0    48M  7280K CPU2     2   0:26  99.52% redis-server{redis-server}
 5212 root         38    0    48M  7280K umtxn    4   0:06  26.94% redis-server{io_thd_3}
 5212 root         36    0    48M  7280K umtxn    6   0:06  26.84% redis-server{io_thd_1}
 5212 root         39    0    48M  7280K umtxn    1   0:06  25.30% redis-server{io_thd_2}
 5212 root         20    0    48M  7280K uwait    3   0:00   0.00% redis-server{redis-server}
 5212 root         21    0    48M  7280K uwait    2   0:00   0.00% redis-server{bio_close_file}
 5212 root         21    0    48M  7280K uwait    3   0:00   0.00% redis-server{bio_aof_fsync}
 5212 root         21    0    48M  7280K uwait    0   0:00   0.00% redis-server{bio_lazy_free}

Signed-off-by: zhenwei pi <pizhenwei@bytedance.com>
---
 src/bio.c        | 12 ++++++++++++
 src/config.h     | 12 ++++++++++++
 src/networking.c |  4 ++++
 3 files changed, 28 insertions(+)

diff --git a/src/bio.c b/src/bio.c
index 2af684570..0662c8c4c 100644
--- a/src/bio.c
+++ b/src/bio.c
@@ -154,6 +154,18 @@ void *bioProcessBackgroundJobs(void *arg) {
         return NULL;
     }
 
+    switch (type) {
+    case BIO_CLOSE_FILE:
+        redis_set_thread_title("bio_close_file");
+        break;
+    case BIO_AOF_FSYNC:
+        redis_set_thread_title("bio_aof_fsync");
+        break;
+    case BIO_LAZY_FREE:
+        redis_set_thread_title("bio_lazy_free");
+        break;
+    }
+
     /* Make the thread killable at any time, so that bioKillThreads()
      * can work reliably. */
     pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL);
diff --git a/src/config.h b/src/config.h
index efa9d11f2..82dc201d3 100644
--- a/src/config.h
+++ b/src/config.h
@@ -226,4 +226,16 @@ void setproctitle(const char *fmt, ...);
 #define USE_ALIGNED_ACCESS
 #endif
 
+/* Define for redis_set_thread_title */
+#ifdef __linux__
+#define redis_set_thread_title(name) pthread_setname_np(pthread_self(), name)
+#else
+#if (defined __NetBSD__ || defined __FreeBSD__ || defined __OpenBSD__)
+#include <pthread_np.h>
+#define redis_set_thread_title(name) pthread_set_name_np(pthread_self(), name)
+#else
+#define redis_set_thread_title(name)
+#endif
+#endif
+
 #endif
diff --git a/src/networking.c b/src/networking.c
index 8f3d79170..1f5d0bd5d 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -2821,6 +2821,10 @@ void *IOThreadMain(void *myid) {
     /* The ID is the thread number (from 0 to server.iothreads_num-1), and is
      * used by the thread to just manipulate a single sub-array of clients. */
     long id = (unsigned long)myid;
+    char thdname[16];
+
+    snprintf(thdname, sizeof(thdname), "io_thd_%ld", id);
+    redis_set_thread_title(thdname);
 
     while(1) {
         /* Wait for start */

From 725b8cc6806dd15f37a7e6b4b828a270c5c96450 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Mon, 20 Apr 2020 12:17:11 +0200
Subject: [PATCH 070/186] Implement redis_set_thread_title for MacOS.

Strange enough, pthread_setname_np() produces a warning for not defined
function even if pthread is included. Moreover the MacOS documentation
claims the return value for the function is void, but actually is int.

Related to #7089.
---
 src/config.h | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/src/config.h b/src/config.h
index 82dc201d3..40dc683ce 100644
--- a/src/config.h
+++ b/src/config.h
@@ -234,8 +234,14 @@ void setproctitle(const char *fmt, ...);
 #include <pthread_np.h>
 #define redis_set_thread_title(name) pthread_set_name_np(pthread_self(), name)
 #else
+#if (defined __APPLE__ && defined(MAC_OS_X_VERSION_10_7))
+int pthread_setname_np(const char *name);
+#include <pthread.h>
+#define redis_set_thread_title(name) pthread_setname_np(name)
+#else
 #define redis_set_thread_title(name)
 #endif
 #endif
+#endif
 
 #endif

From 6791ff0525d21b7517375a1eca3627bf86ff848d Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Tue, 21 Apr 2020 10:51:46 +0200
Subject: [PATCH 071/186] Tracking: NOLOOP internals implementation.

---
 src/bitops.c      |   8 ++--
 src/cluster.c     |   4 +-
 src/db.c          |  30 +++++++-----
 src/debug.c       |   2 +-
 src/expire.c      |   6 +--
 src/geo.c         |   4 +-
 src/hyperloglog.c |   6 +--
 src/module.c      |  10 ++--
 src/server.h      |  12 +++--
 src/sort.c        |   4 +-
 src/t_hash.c      |  10 ++--
 src/t_list.c      |  20 ++++----
 src/t_set.c       |  20 ++++----
 src/t_stream.c    |   6 +--
 src/t_string.c    |  14 +++---
 src/t_zset.c      |  12 ++---
 src/tracking.c    | 118 ++++++++++++++++++++++++++++++++++------------
 17 files changed, 174 insertions(+), 112 deletions(-)

diff --git a/src/bitops.c b/src/bitops.c
index 496d78b66..f506a881b 100644
--- a/src/bitops.c
+++ b/src/bitops.c
@@ -554,7 +554,7 @@ void setbitCommand(client *c) {
     byteval &= ~(1 << bit);
     byteval |= ((on & 0x1) << bit);
     ((uint8_t*)o->ptr)[byte] = byteval;
-    signalModifiedKey(c->db,c->argv[1]);
+    signalModifiedKey(c,c->db,c->argv[1]);
     notifyKeyspaceEvent(NOTIFY_STRING,"setbit",c->argv[1],c->db->id);
     server.dirty++;
     addReply(c, bitval ? shared.cone : shared.czero);
@@ -754,11 +754,11 @@ void bitopCommand(client *c) {
     /* Store the computed value into the target key */
     if (maxlen) {
         o = createObject(OBJ_STRING,res);
-        setKey(c->db,targetkey,o);
+        setKey(c,c->db,targetkey,o);
         notifyKeyspaceEvent(NOTIFY_STRING,"set",targetkey,c->db->id);
         decrRefCount(o);
     } else if (dbDelete(c->db,targetkey)) {
-        signalModifiedKey(c->db,targetkey);
+        signalModifiedKey(c,c->db,targetkey);
         notifyKeyspaceEvent(NOTIFY_GENERIC,"del",targetkey,c->db->id);
     }
     server.dirty++;
@@ -1135,7 +1135,7 @@ void bitfieldGeneric(client *c, int flags) {
     }
 
     if (changes) {
-        signalModifiedKey(c->db,c->argv[1]);
+        signalModifiedKey(c,c->db,c->argv[1]);
         notifyKeyspaceEvent(NOTIFY_STRING,"setbit",c->argv[1],c->db->id);
         server.dirty += changes;
     }
diff --git a/src/cluster.c b/src/cluster.c
index 2377b386b..b103e2fe1 100644
--- a/src/cluster.c
+++ b/src/cluster.c
@@ -4982,7 +4982,7 @@ void restoreCommand(client *c) {
         setExpire(c,c->db,c->argv[1],ttl);
     }
     objectSetLRUOrLFU(obj,lfu_freq,lru_idle,lru_clock,1000);
-    signalModifiedKey(c->db,c->argv[1]);
+    signalModifiedKey(c,c->db,c->argv[1]);
     notifyKeyspaceEvent(NOTIFY_GENERIC,"restore",c->argv[1],c->db->id);
     addReply(c,shared.ok);
     server.dirty++;
@@ -5329,7 +5329,7 @@ try_again:
             if (!copy) {
                 /* No COPY option: remove the local key, signal the change. */
                 dbDelete(c->db,kv[j]);
-                signalModifiedKey(c->db,kv[j]);
+                signalModifiedKey(c,c->db,kv[j]);
                 notifyKeyspaceEvent(NOTIFY_GENERIC,"del",kv[j],c->db->id);
                 server.dirty++;
 
diff --git a/src/db.c b/src/db.c
index 59f0cc7a0..dc4a0b63e 100644
--- a/src/db.c
+++ b/src/db.c
@@ -238,8 +238,10 @@ void dbOverwrite(redisDb *db, robj *key, robj *val) {
  * 3) The expire time of the key is reset (the key is made persistent),
  *    unless 'keepttl' is true.
  *
- * All the new keys in the database should be created via this interface. */
-void genericSetKey(redisDb *db, robj *key, robj *val, int keepttl, int signal) {
+ * All the new keys in the database should be created via this interface.
+ * The client 'c' argument may be set to NULL if the operation is performed
+ * in a context where there is no clear client performing the operation. */
+void genericSetKey(client *c, redisDb *db, robj *key, robj *val, int keepttl, int signal) {
     if (lookupKeyWrite(db,key) == NULL) {
         dbAdd(db,key,val);
     } else {
@@ -247,12 +249,12 @@ void genericSetKey(redisDb *db, robj *key, robj *val, int keepttl, int signal) {
     }
     incrRefCount(val);
     if (!keepttl) removeExpire(db,key);
-    if (signal) signalModifiedKey(db,key);
+    if (signal) signalModifiedKey(c,db,key);
 }
 
 /* Common case for genericSetKey() where the TTL is not retained. */
-void setKey(redisDb *db, robj *key, robj *val) {
-    genericSetKey(db,key,val,0,1);
+void setKey(client *c, redisDb *db, robj *key, robj *val) {
+    genericSetKey(c,db,key,val,0,1);
 }
 
 /* Return true if the specified key exists in the specified database.
@@ -467,9 +469,11 @@ long long dbTotalServerKeyCount() {
  * Every time a DB is flushed the function signalFlushDb() is called.
  *----------------------------------------------------------------------------*/
 
-void signalModifiedKey(redisDb *db, robj *key) {
+/* Note that the 'c' argument may be NULL if the key was modified out of
+ * a context of a client. */
+void signalModifiedKey(client *c, redisDb *db, robj *key) {
     touchWatchedKey(db,key);
-    trackingInvalidateKey(key);
+    trackingInvalidateKey(c,key);
 }
 
 void signalFlushedDb(int dbid) {
@@ -563,7 +567,7 @@ void delGenericCommand(client *c, int lazy) {
         int deleted  = lazy ? dbAsyncDelete(c->db,c->argv[j]) :
                               dbSyncDelete(c->db,c->argv[j]);
         if (deleted) {
-            signalModifiedKey(c->db,c->argv[j]);
+            signalModifiedKey(c,c->db,c->argv[j]);
             notifyKeyspaceEvent(NOTIFY_GENERIC,
                 "del",c->argv[j],c->db->id);
             server.dirty++;
@@ -1003,8 +1007,8 @@ void renameGenericCommand(client *c, int nx) {
     dbAdd(c->db,c->argv[2],o);
     if (expire != -1) setExpire(c,c->db,c->argv[2],expire);
     dbDelete(c->db,c->argv[1]);
-    signalModifiedKey(c->db,c->argv[1]);
-    signalModifiedKey(c->db,c->argv[2]);
+    signalModifiedKey(c,c->db,c->argv[1]);
+    signalModifiedKey(c,c->db,c->argv[2]);
     notifyKeyspaceEvent(NOTIFY_GENERIC,"rename_from",
         c->argv[1],c->db->id);
     notifyKeyspaceEvent(NOTIFY_GENERIC,"rename_to",
@@ -1072,8 +1076,8 @@ void moveCommand(client *c) {
 
     /* OK! key moved, free the entry in the source DB */
     dbDelete(src,c->argv[1]);
-    signalModifiedKey(src,c->argv[1]);
-    signalModifiedKey(dst,c->argv[1]);
+    signalModifiedKey(c,src,c->argv[1]);
+    signalModifiedKey(c,dst,c->argv[1]);
     notifyKeyspaceEvent(NOTIFY_GENERIC,
                 "move_from",c->argv[1],src->id);
     notifyKeyspaceEvent(NOTIFY_GENERIC,
@@ -1317,7 +1321,7 @@ int expireIfNeeded(redisDb *db, robj *key) {
         "expired",key,db->id);
     int retval = server.lazyfree_lazy_expire ? dbAsyncDelete(db,key) :
                                                dbSyncDelete(db,key);
-    if (retval) signalModifiedKey(db,key);
+    if (retval) signalModifiedKey(NULL,db,key);
     return retval;
 }
 
diff --git a/src/debug.c b/src/debug.c
index 1351b2536..cbb56cb71 100644
--- a/src/debug.c
+++ b/src/debug.c
@@ -588,7 +588,7 @@ NULL
                 memcpy(val->ptr, buf, valsize<=buflen? valsize: buflen);
             }
             dbAdd(c->db,key,val);
-            signalModifiedKey(c->db,key);
+            signalModifiedKey(c,c->db,key);
             decrRefCount(key);
         }
         addReply(c,shared.ok);
diff --git a/src/expire.c b/src/expire.c
index c102a01ff..30a27193d 100644
--- a/src/expire.c
+++ b/src/expire.c
@@ -64,7 +64,7 @@ int activeExpireCycleTryExpire(redisDb *db, dictEntry *de, long long now) {
             dbSyncDelete(db,keyobj);
         notifyKeyspaceEvent(NOTIFY_EXPIRED,
             "expired",keyobj,db->id);
-        trackingInvalidateKey(keyobj);
+        trackingInvalidateKey(NULL,keyobj);
         decrRefCount(keyobj);
         server.stat_expiredkeys++;
         return 1;
@@ -519,14 +519,14 @@ void expireGenericCommand(client *c, long long basetime, int unit) {
         /* Replicate/AOF this as an explicit DEL or UNLINK. */
         aux = server.lazyfree_lazy_expire ? shared.unlink : shared.del;
         rewriteClientCommandVector(c,2,aux,key);
-        signalModifiedKey(c->db,key);
+        signalModifiedKey(c,c->db,key);
         notifyKeyspaceEvent(NOTIFY_GENERIC,"del",key,c->db->id);
         addReply(c, shared.cone);
         return;
     } else {
         setExpire(c,c->db,key,when);
         addReply(c,shared.cone);
-        signalModifiedKey(c->db,key);
+        signalModifiedKey(c,c->db,key);
         notifyKeyspaceEvent(NOTIFY_GENERIC,"expire",key,c->db->id);
         server.dirty++;
         return;
diff --git a/src/geo.c b/src/geo.c
index f7920a2e2..3e5d5f606 100644
--- a/src/geo.c
+++ b/src/geo.c
@@ -657,13 +657,13 @@ void georadiusGeneric(client *c, int flags) {
 
         if (returned_items) {
             zsetConvertToZiplistIfNeeded(zobj,maxelelen);
-            setKey(c->db,storekey,zobj);
+            setKey(c,c->db,storekey,zobj);
             decrRefCount(zobj);
             notifyKeyspaceEvent(NOTIFY_ZSET,"georadiusstore",storekey,
                                 c->db->id);
             server.dirty += returned_items;
         } else if (dbDelete(c->db,storekey)) {
-            signalModifiedKey(c->db,storekey);
+            signalModifiedKey(c,c->db,storekey);
             notifyKeyspaceEvent(NOTIFY_GENERIC,"del",storekey,c->db->id);
             server.dirty++;
         }
diff --git a/src/hyperloglog.c b/src/hyperloglog.c
index facd99743..721f492a1 100644
--- a/src/hyperloglog.c
+++ b/src/hyperloglog.c
@@ -1209,7 +1209,7 @@ void pfaddCommand(client *c) {
     }
     hdr = o->ptr;
     if (updated) {
-        signalModifiedKey(c->db,c->argv[1]);
+        signalModifiedKey(c,c->db,c->argv[1]);
         notifyKeyspaceEvent(NOTIFY_STRING,"pfadd",c->argv[1],c->db->id);
         server.dirty++;
         HLL_INVALIDATE_CACHE(hdr);
@@ -1300,7 +1300,7 @@ void pfcountCommand(client *c) {
              * data structure is not modified, since the cached value
              * may be modified and given that the HLL is a Redis string
              * we need to propagate the change. */
-            signalModifiedKey(c->db,c->argv[1]);
+            signalModifiedKey(c,c->db,c->argv[1]);
             server.dirty++;
         }
         addReplyLongLong(c,card);
@@ -1373,7 +1373,7 @@ void pfmergeCommand(client *c) {
                      last hllSparseSet() call. */
     HLL_INVALIDATE_CACHE(hdr);
 
-    signalModifiedKey(c->db,c->argv[1]);
+    signalModifiedKey(c,c->db,c->argv[1]);
     /* We generate a PFADD event for PFMERGE for semantical simplicity
      * since in theory this is a mass-add of elements. */
     notifyKeyspaceEvent(NOTIFY_STRING,"pfadd",c->argv[1],c->db->id);
diff --git a/src/module.c b/src/module.c
index 4d3d9e1af..e3a338dad 100644
--- a/src/module.c
+++ b/src/module.c
@@ -896,7 +896,7 @@ void RM_SetModuleOptions(RedisModuleCtx *ctx, int options) {
 /* Signals that the key is modified from user's perspective (i.e. invalidate WATCH
  * and client side caching). */
 int RM_SignalModifiedKey(RedisModuleCtx *ctx, RedisModuleString *keyname) {
-    signalModifiedKey(ctx->client->db,keyname);
+    signalModifiedKey(ctx->client,ctx->client->db,keyname);
     return REDISMODULE_OK;
 }
 
@@ -2016,7 +2016,7 @@ void *RM_OpenKey(RedisModuleCtx *ctx, robj *keyname, int mode) {
 static void moduleCloseKey(RedisModuleKey *key) {
     int signal = SHOULD_SIGNAL_MODIFIED_KEYS(key->ctx);
     if ((key->mode & REDISMODULE_WRITE) && signal)
-        signalModifiedKey(key->db,key->key);
+        signalModifiedKey(key->ctx->client,key->db,key->key);
     /* TODO: if (key->iter) RM_KeyIteratorStop(kp); */
     RM_ZsetRangeStop(key);
     decrRefCount(key->key);
@@ -2157,7 +2157,7 @@ RedisModuleString *RM_RandomKey(RedisModuleCtx *ctx) {
 int RM_StringSet(RedisModuleKey *key, RedisModuleString *str) {
     if (!(key->mode & REDISMODULE_WRITE) || key->iter) return REDISMODULE_ERR;
     RM_DeleteKey(key);
-    genericSetKey(key->db,key->key,str,0,0);
+    genericSetKey(key->ctx->client,key->db,key->key,str,0,0);
     key->value = str;
     return REDISMODULE_OK;
 }
@@ -2237,7 +2237,7 @@ int RM_StringTruncate(RedisModuleKey *key, size_t newlen) {
     if (key->value == NULL) {
         /* Empty key: create it with the new size. */
         robj *o = createObject(OBJ_STRING,sdsnewlen(NULL, newlen));
-        genericSetKey(key->db,key->key,o,0,0);
+        genericSetKey(key->ctx->client,key->db,key->key,o,0,0);
         key->value = o;
         decrRefCount(o);
     } else {
@@ -3625,7 +3625,7 @@ int RM_ModuleTypeSetValue(RedisModuleKey *key, moduleType *mt, void *value) {
     if (!(key->mode & REDISMODULE_WRITE) || key->iter) return REDISMODULE_ERR;
     RM_DeleteKey(key);
     robj *o = createModuleObject(mt,value);
-    genericSetKey(key->db,key->key,o,0,0);
+    genericSetKey(key->ctx->client,key->db,key->key,o,0,0);
     decrRefCount(o);
     key->value = o;
     return REDISMODULE_OK;
diff --git a/src/server.h b/src/server.h
index d0d5ff154..d39359dce 100644
--- a/src/server.h
+++ b/src/server.h
@@ -252,7 +252,9 @@ typedef long long ustime_t; /* microsecond time type. */
 #define CLIENT_TRACKING_OPTOUT (1ULL<<35) /* Tracking in opt-out mode. */
 #define CLIENT_TRACKING_CACHING (1ULL<<36) /* CACHING yes/no was given,
                                               depending on optin/optout mode. */
-#define CLIENT_IN_TO_TABLE (1ULL<<37) /* This client is in the timeout table. */
+#define CLIENT_TRACKING_NOLOOP (1ULL<<37) /* Don't send invalidation messages
+                                             about writes performed by myself.*/
+#define CLIENT_IN_TO_TABLE (1ULL<<38) /* This client is in the timeout table. */
 
 /* Client block type (btype field in client structure)
  * if CLIENT_BLOCKED flag is set. */
@@ -1683,7 +1685,7 @@ void addReplyStatusFormat(client *c, const char *fmt, ...);
 void enableTracking(client *c, uint64_t redirect_to, uint64_t options, robj **prefix, size_t numprefix);
 void disableTracking(client *c);
 void trackingRememberKeys(client *c);
-void trackingInvalidateKey(robj *keyobj);
+void trackingInvalidateKey(client *c, robj *keyobj);
 void trackingInvalidateKeysOnFlush(int dbid);
 void trackingLimitUsedSlots(void);
 uint64_t trackingGetTotalItems(void);
@@ -2071,8 +2073,8 @@ int objectSetLRUOrLFU(robj *val, long long lfu_freq, long long lru_idle,
 void dbAdd(redisDb *db, robj *key, robj *val);
 int dbAddRDBLoad(redisDb *db, sds key, robj *val);
 void dbOverwrite(redisDb *db, robj *key, robj *val);
-void genericSetKey(redisDb *db, robj *key, robj *val, int keepttl, int signal);
-void setKey(redisDb *db, robj *key, robj *val);
+void genericSetKey(client *c, redisDb *db, robj *key, robj *val, int keepttl, int signal);
+void setKey(client *c, redisDb *db, robj *key, robj *val);
 int dbExists(redisDb *db, robj *key);
 robj *dbRandomKey(redisDb *db);
 int dbSyncDelete(redisDb *db, robj *key);
@@ -2088,7 +2090,7 @@ void flushAllDataAndResetRDB(int flags);
 long long dbTotalServerKeyCount();
 
 int selectDb(client *c, int id);
-void signalModifiedKey(redisDb *db, robj *key);
+void signalModifiedKey(client *c, redisDb *db, robj *key);
 void signalFlushedDb(int dbid);
 unsigned int getKeysInSlot(unsigned int hashslot, robj **keys, unsigned int count);
 unsigned int countKeysInSlot(unsigned int hashslot);
diff --git a/src/sort.c b/src/sort.c
index db26da158..f269a7731 100644
--- a/src/sort.c
+++ b/src/sort.c
@@ -570,12 +570,12 @@ void sortCommand(client *c) {
             }
         }
         if (outputlen) {
-            setKey(c->db,storekey,sobj);
+            setKey(c,c->db,storekey,sobj);
             notifyKeyspaceEvent(NOTIFY_LIST,"sortstore",storekey,
                                 c->db->id);
             server.dirty += outputlen;
         } else if (dbDelete(c->db,storekey)) {
-            signalModifiedKey(c->db,storekey);
+            signalModifiedKey(c,c->db,storekey);
             notifyKeyspaceEvent(NOTIFY_GENERIC,"del",storekey,c->db->id);
             server.dirty++;
         }
diff --git a/src/t_hash.c b/src/t_hash.c
index b9f0db7fc..866bcd25b 100644
--- a/src/t_hash.c
+++ b/src/t_hash.c
@@ -521,7 +521,7 @@ void hsetnxCommand(client *c) {
     } else {
         hashTypeSet(o,c->argv[2]->ptr,c->argv[3]->ptr,HASH_SET_COPY);
         addReply(c, shared.cone);
-        signalModifiedKey(c->db,c->argv[1]);
+        signalModifiedKey(c,c->db,c->argv[1]);
         notifyKeyspaceEvent(NOTIFY_HASH,"hset",c->argv[1],c->db->id);
         server.dirty++;
     }
@@ -551,7 +551,7 @@ void hsetCommand(client *c) {
         /* HMSET */
         addReply(c, shared.ok);
     }
-    signalModifiedKey(c->db,c->argv[1]);
+    signalModifiedKey(c,c->db,c->argv[1]);
     notifyKeyspaceEvent(NOTIFY_HASH,"hset",c->argv[1],c->db->id);
     server.dirty++;
 }
@@ -586,7 +586,7 @@ void hincrbyCommand(client *c) {
     new = sdsfromlonglong(value);
     hashTypeSet(o,c->argv[2]->ptr,new,HASH_SET_TAKE_VALUE);
     addReplyLongLong(c,value);
-    signalModifiedKey(c->db,c->argv[1]);
+    signalModifiedKey(c,c->db,c->argv[1]);
     notifyKeyspaceEvent(NOTIFY_HASH,"hincrby",c->argv[1],c->db->id);
     server.dirty++;
 }
@@ -625,7 +625,7 @@ void hincrbyfloatCommand(client *c) {
     new = sdsnewlen(buf,len);
     hashTypeSet(o,c->argv[2]->ptr,new,HASH_SET_TAKE_VALUE);
     addReplyBulkCBuffer(c,buf,len);
-    signalModifiedKey(c->db,c->argv[1]);
+    signalModifiedKey(c,c->db,c->argv[1]);
     notifyKeyspaceEvent(NOTIFY_HASH,"hincrbyfloat",c->argv[1],c->db->id);
     server.dirty++;
 
@@ -721,7 +721,7 @@ void hdelCommand(client *c) {
         }
     }
     if (deleted) {
-        signalModifiedKey(c->db,c->argv[1]);
+        signalModifiedKey(c,c->db,c->argv[1]);
         notifyKeyspaceEvent(NOTIFY_HASH,"hdel",c->argv[1],c->db->id);
         if (keyremoved)
             notifyKeyspaceEvent(NOTIFY_GENERIC,"del",c->argv[1],
diff --git a/src/t_list.c b/src/t_list.c
index eaeaa8e48..4770a2272 100644
--- a/src/t_list.c
+++ b/src/t_list.c
@@ -217,7 +217,7 @@ void pushGenericCommand(client *c, int where) {
     if (pushed) {
         char *event = (where == LIST_HEAD) ? "lpush" : "rpush";
 
-        signalModifiedKey(c->db,c->argv[1]);
+        signalModifiedKey(c,c->db,c->argv[1]);
         notifyKeyspaceEvent(NOTIFY_LIST,event,c->argv[1],c->db->id);
     }
     server.dirty += pushed;
@@ -247,7 +247,7 @@ void pushxGenericCommand(client *c, int where) {
 
     if (pushed) {
         char *event = (where == LIST_HEAD) ? "lpush" : "rpush";
-        signalModifiedKey(c->db,c->argv[1]);
+        signalModifiedKey(c,c->db,c->argv[1]);
         notifyKeyspaceEvent(NOTIFY_LIST,event,c->argv[1],c->db->id);
     }
     server.dirty += pushed;
@@ -292,7 +292,7 @@ void linsertCommand(client *c) {
     listTypeReleaseIterator(iter);
 
     if (inserted) {
-        signalModifiedKey(c->db,c->argv[1]);
+        signalModifiedKey(c,c->db,c->argv[1]);
         notifyKeyspaceEvent(NOTIFY_LIST,"linsert",
                             c->argv[1],c->db->id);
         server.dirty++;
@@ -355,7 +355,7 @@ void lsetCommand(client *c) {
             addReply(c,shared.outofrangeerr);
         } else {
             addReply(c,shared.ok);
-            signalModifiedKey(c->db,c->argv[1]);
+            signalModifiedKey(c,c->db,c->argv[1]);
             notifyKeyspaceEvent(NOTIFY_LIST,"lset",c->argv[1],c->db->id);
             server.dirty++;
         }
@@ -382,7 +382,7 @@ void popGenericCommand(client *c, int where) {
                                 c->argv[1],c->db->id);
             dbDelete(c->db,c->argv[1]);
         }
-        signalModifiedKey(c->db,c->argv[1]);
+        signalModifiedKey(c,c->db,c->argv[1]);
         server.dirty++;
     }
 }
@@ -482,7 +482,7 @@ void ltrimCommand(client *c) {
         dbDelete(c->db,c->argv[1]);
         notifyKeyspaceEvent(NOTIFY_GENERIC,"del",c->argv[1],c->db->id);
     }
-    signalModifiedKey(c->db,c->argv[1]);
+    signalModifiedKey(c,c->db,c->argv[1]);
     server.dirty++;
     addReply(c,shared.ok);
 }
@@ -519,7 +519,7 @@ void lremCommand(client *c) {
     listTypeReleaseIterator(li);
 
     if (removed) {
-        signalModifiedKey(c->db,c->argv[1]);
+        signalModifiedKey(c,c->db,c->argv[1]);
         notifyKeyspaceEvent(NOTIFY_LIST,"lrem",c->argv[1],c->db->id);
     }
 
@@ -555,7 +555,7 @@ void rpoplpushHandlePush(client *c, robj *dstkey, robj *dstobj, robj *value) {
                             server.list_compress_depth);
         dbAdd(c->db,dstkey,dstobj);
     }
-    signalModifiedKey(c->db,dstkey);
+    signalModifiedKey(c,c->db,dstkey);
     listTypePush(dstobj,value,LIST_HEAD);
     notifyKeyspaceEvent(NOTIFY_LIST,"lpush",dstkey,c->db->id);
     /* Always send the pushed value to the client. */
@@ -593,7 +593,7 @@ void rpoplpushCommand(client *c) {
             notifyKeyspaceEvent(NOTIFY_GENERIC,"del",
                                 touchedkey,c->db->id);
         }
-        signalModifiedKey(c->db,touchedkey);
+        signalModifiedKey(c,c->db,touchedkey);
         decrRefCount(touchedkey);
         server.dirty++;
         if (c->cmd->proc == brpoplpushCommand) {
@@ -708,7 +708,7 @@ void blockingPopGenericCommand(client *c, int where) {
                         notifyKeyspaceEvent(NOTIFY_GENERIC,"del",
                                             c->argv[j],c->db->id);
                     }
-                    signalModifiedKey(c->db,c->argv[j]);
+                    signalModifiedKey(c,c->db,c->argv[j]);
                     server.dirty++;
 
                     /* Replicate it as an [LR]POP instead of B[LR]POP. */
diff --git a/src/t_set.c b/src/t_set.c
index 60cf22d8c..c2e73a6e6 100644
--- a/src/t_set.c
+++ b/src/t_set.c
@@ -280,7 +280,7 @@ void saddCommand(client *c) {
         if (setTypeAdd(set,c->argv[j]->ptr)) added++;
     }
     if (added) {
-        signalModifiedKey(c->db,c->argv[1]);
+        signalModifiedKey(c,c->db,c->argv[1]);
         notifyKeyspaceEvent(NOTIFY_SET,"sadd",c->argv[1],c->db->id);
     }
     server.dirty += added;
@@ -305,7 +305,7 @@ void sremCommand(client *c) {
         }
     }
     if (deleted) {
-        signalModifiedKey(c->db,c->argv[1]);
+        signalModifiedKey(c,c->db,c->argv[1]);
         notifyKeyspaceEvent(NOTIFY_SET,"srem",c->argv[1],c->db->id);
         if (keyremoved)
             notifyKeyspaceEvent(NOTIFY_GENERIC,"del",c->argv[1],
@@ -358,8 +358,8 @@ void smoveCommand(client *c) {
         dbAdd(c->db,c->argv[2],dstset);
     }
 
-    signalModifiedKey(c->db,c->argv[1]);
-    signalModifiedKey(c->db,c->argv[2]);
+    signalModifiedKey(c,c->db,c->argv[1]);
+    signalModifiedKey(c,c->db,c->argv[2]);
     server.dirty++;
 
     /* An extra key has changed when ele was successfully added to dstset */
@@ -444,7 +444,7 @@ void spopWithCountCommand(client *c) {
 
         /* Propagate this command as an DEL operation */
         rewriteClientCommandVector(c,2,shared.del,c->argv[1]);
-        signalModifiedKey(c->db,c->argv[1]);
+        signalModifiedKey(c,c->db,c->argv[1]);
         server.dirty++;
         return;
     }
@@ -546,7 +546,7 @@ void spopWithCountCommand(client *c) {
      * the alsoPropagate() API. */
     decrRefCount(propargv[0]);
     preventCommandPropagation(c);
-    signalModifiedKey(c->db,c->argv[1]);
+    signalModifiedKey(c,c->db,c->argv[1]);
     server.dirty++;
 }
 
@@ -599,7 +599,7 @@ void spopCommand(client *c) {
     }
 
     /* Set has been modified */
-    signalModifiedKey(c->db,c->argv[1]);
+    signalModifiedKey(c,c->db,c->argv[1]);
     server.dirty++;
 }
 
@@ -808,7 +808,7 @@ void sinterGenericCommand(client *c, robj **setkeys,
             zfree(sets);
             if (dstkey) {
                 if (dbDelete(c->db,dstkey)) {
-                    signalModifiedKey(c->db,dstkey);
+                    signalModifiedKey(c,c->db,dstkey);
                     server.dirty++;
                 }
                 addReply(c,shared.czero);
@@ -908,7 +908,7 @@ void sinterGenericCommand(client *c, robj **setkeys,
                 notifyKeyspaceEvent(NOTIFY_GENERIC,"del",
                     dstkey,c->db->id);
         }
-        signalModifiedKey(c->db,dstkey);
+        signalModifiedKey(c,c->db,dstkey);
         server.dirty++;
     } else {
         setDeferredSetLen(c,replylen,cardinality);
@@ -1083,7 +1083,7 @@ void sunionDiffGenericCommand(client *c, robj **setkeys, int setnum,
                 notifyKeyspaceEvent(NOTIFY_GENERIC,"del",
                     dstkey,c->db->id);
         }
-        signalModifiedKey(c->db,dstkey);
+        signalModifiedKey(c,c->db,dstkey);
         server.dirty++;
     }
     zfree(sets);
diff --git a/src/t_stream.c b/src/t_stream.c
index 155167af9..3efaa4509 100644
--- a/src/t_stream.c
+++ b/src/t_stream.c
@@ -1262,7 +1262,7 @@ void xaddCommand(client *c) {
     }
     addReplyStreamID(c,&id);
 
-    signalModifiedKey(c->db,c->argv[1]);
+    signalModifiedKey(c,c->db,c->argv[1]);
     notifyKeyspaceEvent(NOTIFY_STREAM,"xadd",c->argv[1],c->db->id);
     server.dirty++;
 
@@ -2390,7 +2390,7 @@ void xdelCommand(client *c) {
 
     /* Propagate the write if needed. */
     if (deleted) {
-        signalModifiedKey(c->db,c->argv[1]);
+        signalModifiedKey(c,c->db,c->argv[1]);
         notifyKeyspaceEvent(NOTIFY_STREAM,"xdel",c->argv[1],c->db->id);
         server.dirty += deleted;
     }
@@ -2467,7 +2467,7 @@ void xtrimCommand(client *c) {
 
     /* Propagate the write if needed. */
     if (deleted) {
-        signalModifiedKey(c->db,c->argv[1]);
+        signalModifiedKey(c,c->db,c->argv[1]);
         notifyKeyspaceEvent(NOTIFY_STREAM,"xtrim",c->argv[1],c->db->id);
         server.dirty += deleted;
         if (approx_maxlen) streamRewriteApproxMaxlen(c,s,maxlen_arg_idx);
diff --git a/src/t_string.c b/src/t_string.c
index 335bda404..ef382bb0c 100644
--- a/src/t_string.c
+++ b/src/t_string.c
@@ -84,7 +84,7 @@ void setGenericCommand(client *c, int flags, robj *key, robj *val, robj *expire,
         addReply(c, abort_reply ? abort_reply : shared.null[c->resp]);
         return;
     }
-    genericSetKey(c->db,key,val,flags & OBJ_SET_KEEPTTL,1);
+    genericSetKey(c,c->db,key,val,flags & OBJ_SET_KEEPTTL,1);
     server.dirty++;
     if (expire) setExpire(c,c->db,key,mstime()+milliseconds);
     notifyKeyspaceEvent(NOTIFY_STRING,"set",key,c->db->id);
@@ -183,7 +183,7 @@ void getCommand(client *c) {
 void getsetCommand(client *c) {
     if (getGenericCommand(c) == C_ERR) return;
     c->argv[2] = tryObjectEncoding(c->argv[2]);
-    setKey(c->db,c->argv[1],c->argv[2]);
+    setKey(c,c->db,c->argv[1],c->argv[2]);
     notifyKeyspaceEvent(NOTIFY_STRING,"set",c->argv[1],c->db->id);
     server.dirty++;
 }
@@ -240,7 +240,7 @@ void setrangeCommand(client *c) {
     if (sdslen(value) > 0) {
         o->ptr = sdsgrowzero(o->ptr,offset+sdslen(value));
         memcpy((char*)o->ptr+offset,value,sdslen(value));
-        signalModifiedKey(c->db,c->argv[1]);
+        signalModifiedKey(c,c->db,c->argv[1]);
         notifyKeyspaceEvent(NOTIFY_STRING,
             "setrange",c->argv[1],c->db->id);
         server.dirty++;
@@ -328,7 +328,7 @@ void msetGenericCommand(client *c, int nx) {
 
     for (j = 1; j < c->argc; j += 2) {
         c->argv[j+1] = tryObjectEncoding(c->argv[j+1]);
-        setKey(c->db,c->argv[j],c->argv[j+1]);
+        setKey(c,c->db,c->argv[j],c->argv[j+1]);
         notifyKeyspaceEvent(NOTIFY_STRING,"set",c->argv[j],c->db->id);
     }
     server.dirty += (c->argc-1)/2;
@@ -373,7 +373,7 @@ void incrDecrCommand(client *c, long long incr) {
             dbAdd(c->db,c->argv[1],new);
         }
     }
-    signalModifiedKey(c->db,c->argv[1]);
+    signalModifiedKey(c,c->db,c->argv[1]);
     notifyKeyspaceEvent(NOTIFY_STRING,"incrby",c->argv[1],c->db->id);
     server.dirty++;
     addReply(c,shared.colon);
@@ -423,7 +423,7 @@ void incrbyfloatCommand(client *c) {
         dbOverwrite(c->db,c->argv[1],new);
     else
         dbAdd(c->db,c->argv[1],new);
-    signalModifiedKey(c->db,c->argv[1]);
+    signalModifiedKey(c,c->db,c->argv[1]);
     notifyKeyspaceEvent(NOTIFY_STRING,"incrbyfloat",c->argv[1],c->db->id);
     server.dirty++;
     addReplyBulk(c,new);
@@ -467,7 +467,7 @@ void appendCommand(client *c) {
         o->ptr = sdscatlen(o->ptr,append->ptr,sdslen(append->ptr));
         totlen = sdslen(o->ptr);
     }
-    signalModifiedKey(c->db,c->argv[1]);
+    signalModifiedKey(c,c->db,c->argv[1]);
     notifyKeyspaceEvent(NOTIFY_STRING,"append",c->argv[1],c->db->id);
     server.dirty++;
     addReplyLongLong(c,totlen);
diff --git a/src/t_zset.c b/src/t_zset.c
index 5c000e76f..9c409cd96 100644
--- a/src/t_zset.c
+++ b/src/t_zset.c
@@ -1646,7 +1646,7 @@ reply_to_client:
 cleanup:
     zfree(scores);
     if (added || updated) {
-        signalModifiedKey(c->db,key);
+        signalModifiedKey(c,c->db,key);
         notifyKeyspaceEvent(NOTIFY_ZSET,
             incr ? "zincr" : "zadd", key, c->db->id);
     }
@@ -1681,7 +1681,7 @@ void zremCommand(client *c) {
         notifyKeyspaceEvent(NOTIFY_ZSET,"zrem",key,c->db->id);
         if (keyremoved)
             notifyKeyspaceEvent(NOTIFY_GENERIC,"del",key,c->db->id);
-        signalModifiedKey(c->db,key);
+        signalModifiedKey(c,c->db,key);
         server.dirty += deleted;
     }
     addReplyLongLong(c,deleted);
@@ -1779,7 +1779,7 @@ void zremrangeGenericCommand(client *c, int rangetype) {
     /* Step 4: Notifications and reply. */
     if (deleted) {
         char *event[3] = {"zremrangebyrank","zremrangebyscore","zremrangebylex"};
-        signalModifiedKey(c->db,key);
+        signalModifiedKey(c,c->db,key);
         notifyKeyspaceEvent(NOTIFY_ZSET,event[rangetype],key,c->db->id);
         if (keyremoved)
             notifyKeyspaceEvent(NOTIFY_GENERIC,"del",key,c->db->id);
@@ -2383,7 +2383,7 @@ void zunionInterGenericCommand(client *c, robj *dstkey, int op) {
         zsetConvertToZiplistIfNeeded(dstobj,maxelelen);
         dbAdd(c->db,dstkey,dstobj);
         addReplyLongLong(c,zsetLength(dstobj));
-        signalModifiedKey(c->db,dstkey);
+        signalModifiedKey(c,c->db,dstkey);
         notifyKeyspaceEvent(NOTIFY_ZSET,
             (op == SET_OP_UNION) ? "zunionstore" : "zinterstore",
             dstkey,c->db->id);
@@ -2392,7 +2392,7 @@ void zunionInterGenericCommand(client *c, robj *dstkey, int op) {
         decrRefCount(dstobj);
         addReply(c,shared.czero);
         if (touched) {
-            signalModifiedKey(c->db,dstkey);
+            signalModifiedKey(c,c->db,dstkey);
             notifyKeyspaceEvent(NOTIFY_GENERIC,"del",dstkey,c->db->id);
             server.dirty++;
         }
@@ -3216,7 +3216,7 @@ void genericZpopCommand(client *c, robj **keyv, int keyc, int where, int emitkey
         if (arraylen == 0) { /* Do this only for the first iteration. */
             char *events[2] = {"zpopmin","zpopmax"};
             notifyKeyspaceEvent(NOTIFY_ZSET,events[where],key,c->db->id);
-            signalModifiedKey(c->db,key);
+            signalModifiedKey(c,c->db,key);
         }
 
         addReplyBulkCBuffer(c,ele,sdslen(ele));
diff --git a/src/tracking.c b/src/tracking.c
index 6f7929430..434e086b5 100644
--- a/src/tracking.c
+++ b/src/tracking.c
@@ -245,7 +245,7 @@ void sendTrackingMessage(client *c, char *keyname, size_t keylen, int proto) {
  * matches one or more prefixes in the prefix table. Later when we
  * return to the event loop, we'll send invalidation messages to the
  * clients subscribed to each prefix. */
-void trackingRememberKeyToBroadcast(char *keyname, size_t keylen) {
+void trackingRememberKeyToBroadcast(client *c, char *keyname, size_t keylen) {
     raxIterator ri;
     raxStart(&ri,PrefixTable);
     raxSeek(&ri,"^",NULL,0);
@@ -254,7 +254,11 @@ void trackingRememberKeyToBroadcast(char *keyname, size_t keylen) {
         if (ri.key_len != 0 && memcmp(ri.key,keyname,ri.key_len) != 0)
             continue;
         bcastState *bs = ri.data;
-        raxTryInsert(bs->keys,(unsigned char*)keyname,keylen,NULL,NULL);
+        /* We insert the client pointer as associated value in the radix
+         * tree. This way we know who was the client that did the last
+         * change to the key, and can avoid sending the notification in the
+         * case the client is in NOLOOP mode. */
+        raxTryInsert(bs->keys,(unsigned char*)keyname,keylen,c,NULL);
     }
     raxStop(&ri);
 }
@@ -262,13 +266,17 @@ void trackingRememberKeyToBroadcast(char *keyname, size_t keylen) {
 /* This function is called from signalModifiedKey() or other places in Redis
  * when a key changes value. In the context of keys tracking, our task here is
  * to send a notification to every client that may have keys about such caching
- * slot. */
-void trackingInvalidateKey(robj *keyobj) {
+ * slot.
+ *
+ * Note that 'c' may be NULL in case the operation was performed outside the
+ * context of a client modifying the database (for instance when we delete a
+ * key because of expire). */
+void trackingInvalidateKey(client *c, robj *keyobj) {
     if (TrackingTable == NULL) return;
     sds sdskey = keyobj->ptr;
 
     if (raxSize(PrefixTable) > 0)
-        trackingRememberKeyToBroadcast(sdskey,sdslen(sdskey));
+        trackingRememberKeyToBroadcast(c,sdskey,sdslen(sdskey));
 
     rax *ids = raxFind(TrackingTable,(unsigned char*)sdskey,sdslen(sdskey));
     if (ids == raxNotFound) return;
@@ -279,19 +287,28 @@ void trackingInvalidateKey(robj *keyobj) {
     while(raxNext(&ri)) {
         uint64_t id;
         memcpy(&id,ri.key,sizeof(id));
-        client *c = lookupClientByID(id);
+        client *target = lookupClientByID(id);
         /* Note that if the client is in BCAST mode, we don't want to
          * send invalidation messages that were pending in the case
          * previously the client was not in BCAST mode. This can happen if
          * TRACKING is enabled normally, and then the client switches to
          * BCAST mode. */
-        if (c == NULL ||
-            !(c->flags & CLIENT_TRACKING)||
-            c->flags & CLIENT_TRACKING_BCAST)
+        if (target == NULL ||
+            !(target->flags & CLIENT_TRACKING)||
+            target->flags & CLIENT_TRACKING_BCAST)
         {
             continue;
         }
-        sendTrackingMessage(c,sdskey,sdslen(sdskey),0);
+
+        /* If the client enabled the NOLOOP mode, don't send notifications
+         * about keys changed by the client itself. */
+        if (target->flags & CLIENT_TRACKING_NOLOOP &&
+            target == c)
+        {
+            continue;
+        }
+
+        sendTrackingMessage(target,sdskey,sdslen(sdskey),0);
     }
     raxStop(&ri);
 
@@ -383,6 +400,54 @@ void trackingLimitUsedSlots(void) {
     timeout_counter++;
 }
 
+/* Generate Redis protocol for an array containing all the key names
+ * in the 'keys' radix tree. If the client is not NULL, the list will not
+ * include keys that were modified the last time by this client, in order
+ * to implement the NOLOOP option.
+ *
+ * If the resultin array would be empty, NULL is returned instead. */
+sds trackingBuildBroadcastReply(client *c, rax *keys) {
+    raxIterator ri;
+    uint64_t count;
+
+    if (c == NULL) {
+        count = raxSize(keys);
+    } else {
+        count = 0;
+        raxStart(&ri,keys);
+        raxSeek(&ri,"^",NULL,0);
+        while(raxNext(&ri)) {
+            if (ri.data != c) count++;
+        }
+        raxStop(&ri);
+
+        if (count == 0) return NULL;
+    }
+
+    /* Create the array reply with the list of keys once, then send
+    * it to all the clients subscribed to this prefix. */
+    char buf[32];
+    size_t len = ll2string(buf,sizeof(buf),count);
+    sds proto = sdsempty();
+    proto = sdsMakeRoomFor(proto,count*15);
+    proto = sdscatlen(proto,"*",1);
+    proto = sdscatlen(proto,buf,len);
+    proto = sdscatlen(proto,"\r\n",2);
+    raxStart(&ri,keys);
+    raxSeek(&ri,"^",NULL,0);
+    while(raxNext(&ri)) {
+        if (c && ri.data == c) continue;
+        len = ll2string(buf,sizeof(buf),ri.key_len);
+        proto = sdscatlen(proto,"$",1);
+        proto = sdscatlen(proto,buf,len);
+        proto = sdscatlen(proto,"\r\n",2);
+        proto = sdscatlen(proto,ri.key,ri.key_len);
+        proto = sdscatlen(proto,"\r\n",2);
+    }
+    raxStop(&ri);
+    return proto;
+}
+
 /* This function will run the prefixes of clients in BCAST mode and
  * keys that were modified about each prefix, and will send the
  * notifications to each client in each prefix. */
@@ -397,26 +462,10 @@ void trackingBroadcastInvalidationMessages(void) {
     while(raxNext(&ri)) {
         bcastState *bs = ri.data;
         if (raxSize(bs->keys)) {
-            /* Create the array reply with the list of keys once, then send
-            * it to all the clients subscribed to this prefix. */
-            char buf[32];
-            size_t len = ll2string(buf,sizeof(buf),raxSize(bs->keys));
-            sds proto = sdsempty();
-            proto = sdsMakeRoomFor(proto,raxSize(bs->keys)*15);
-            proto = sdscatlen(proto,"*",1);
-            proto = sdscatlen(proto,buf,len);
-            proto = sdscatlen(proto,"\r\n",2);
-            raxStart(&ri2,bs->keys);
-            raxSeek(&ri2,"^",NULL,0);
-            while(raxNext(&ri2)) {
-                len = ll2string(buf,sizeof(buf),ri2.key_len);
-                proto = sdscatlen(proto,"$",1);
-                proto = sdscatlen(proto,buf,len);
-                proto = sdscatlen(proto,"\r\n",2);
-                proto = sdscatlen(proto,ri2.key,ri2.key_len);
-                proto = sdscatlen(proto,"\r\n",2);
-            }
-            raxStop(&ri2);
+
+            /* Generate the common protocol for all the clients that are
+             * not using the NOLOOP option. */
+            sds proto = trackingBuildBroadcastReply(NULL,bs->keys);
 
             /* Send this array of keys to every client in the list. */
             raxStart(&ri2,bs->clients);
@@ -424,7 +473,14 @@ void trackingBroadcastInvalidationMessages(void) {
             while(raxNext(&ri2)) {
                 client *c;
                 memcpy(&c,ri2.key,sizeof(c));
-                sendTrackingMessage(c,proto,sdslen(proto),1);
+                if (c->flags & CLIENT_TRACKING_NOLOOP) {
+                    /* This client may have certain keys excluded. */
+                    sds adhoc = trackingBuildBroadcastReply(c,bs->keys);
+                    sendTrackingMessage(c,adhoc,sdslen(adhoc),1);
+                    sdsfree(adhoc);
+                } else {
+                    sendTrackingMessage(c,proto,sdslen(proto),1);
+                }
             }
             raxStop(&ri2);
 

From e63bb7ec8cebff00e0236ac90e7dc2242b8189aa Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Tue, 21 Apr 2020 17:29:18 +0200
Subject: [PATCH 072/186] Tracking: NOLOOP further implementation and fixes.

---
 src/networking.c |  2 ++
 src/tracking.c   | 25 +++++++++++++++++++------
 2 files changed, 21 insertions(+), 6 deletions(-)

diff --git a/src/networking.c b/src/networking.c
index 1f5d0bd5d..744979d16 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -2289,6 +2289,8 @@ NULL
                 options |= CLIENT_TRACKING_OPTIN;
             } else if (!strcasecmp(c->argv[j]->ptr,"optout")) {
                 options |= CLIENT_TRACKING_OPTOUT;
+            } else if (!strcasecmp(c->argv[j]->ptr,"noloop")) {
+                options |= CLIENT_TRACKING_NOLOOP;
             } else if (!strcasecmp(c->argv[j]->ptr,"prefix") && moreargs) {
                 j++;
                 prefix = zrealloc(prefix,sizeof(robj*)*(numprefix+1));
diff --git a/src/tracking.c b/src/tracking.c
index 434e086b5..48d231627 100644
--- a/src/tracking.c
+++ b/src/tracking.c
@@ -94,7 +94,8 @@ void disableTracking(client *c) {
         server.tracking_clients--;
         c->flags &= ~(CLIENT_TRACKING|CLIENT_TRACKING_BROKEN_REDIR|
                       CLIENT_TRACKING_BCAST|CLIENT_TRACKING_OPTIN|
-                      CLIENT_TRACKING_OPTOUT|CLIENT_TRACKING_CACHING);
+                      CLIENT_TRACKING_OPTOUT|CLIENT_TRACKING_CACHING|
+                      CLIENT_TRACKING_NOLOOP);
     }
 }
 
@@ -129,14 +130,19 @@ void enableTracking(client *c, uint64_t redirect_to, uint64_t options, robj **pr
     if (!(c->flags & CLIENT_TRACKING)) server.tracking_clients++;
     c->flags |= CLIENT_TRACKING;
     c->flags &= ~(CLIENT_TRACKING_BROKEN_REDIR|CLIENT_TRACKING_BCAST|
-                  CLIENT_TRACKING_OPTIN|CLIENT_TRACKING_OPTOUT);
+                  CLIENT_TRACKING_OPTIN|CLIENT_TRACKING_OPTOUT|
+                  CLIENT_TRACKING_NOLOOP);
     c->client_tracking_redirection = redirect_to;
+
+    /* This may be the first client we ever enable. Crete the tracking
+     * table if it does not exist. */
     if (TrackingTable == NULL) {
         TrackingTable = raxNew();
         PrefixTable = raxNew();
         TrackingChannelName = createStringObject("__redis__:invalidate",20);
     }
 
+    /* For broadcasting, set the list of prefixes in the client. */
     if (options & CLIENT_TRACKING_BCAST) {
         c->flags |= CLIENT_TRACKING_BCAST;
         if (numprefix == 0) enableBcastTrackingForPrefix(c,"",0);
@@ -145,7 +151,10 @@ void enableTracking(client *c, uint64_t redirect_to, uint64_t options, robj **pr
             enableBcastTrackingForPrefix(c,sdsprefix,sdslen(sdsprefix));
         }
     }
-    c->flags |= options & (CLIENT_TRACKING_OPTIN|CLIENT_TRACKING_OPTOUT);
+
+    /* Set the remaining flags that don't need any special handling. */
+    c->flags |= options & (CLIENT_TRACKING_OPTIN|CLIENT_TRACKING_OPTOUT|
+                           CLIENT_TRACKING_NOLOOP);
 }
 
 /* This function is called after the execution of a readonly command in the
@@ -459,10 +468,12 @@ void trackingBroadcastInvalidationMessages(void) {
 
     raxStart(&ri,PrefixTable);
     raxSeek(&ri,"^",NULL,0);
+
+    /* For each prefix... */
     while(raxNext(&ri)) {
         bcastState *bs = ri.data;
-        if (raxSize(bs->keys)) {
 
+        if (raxSize(bs->keys)) {
             /* Generate the common protocol for all the clients that are
              * not using the NOLOOP option. */
             sds proto = trackingBuildBroadcastReply(NULL,bs->keys);
@@ -476,8 +487,10 @@ void trackingBroadcastInvalidationMessages(void) {
                 if (c->flags & CLIENT_TRACKING_NOLOOP) {
                     /* This client may have certain keys excluded. */
                     sds adhoc = trackingBuildBroadcastReply(c,bs->keys);
-                    sendTrackingMessage(c,adhoc,sdslen(adhoc),1);
-                    sdsfree(adhoc);
+                    if (adhoc) {
+                        sendTrackingMessage(c,adhoc,sdslen(adhoc),1);
+                        sdsfree(adhoc);
+                    }
                 } else {
                     sendTrackingMessage(c,proto,sdslen(proto),1);
                 }

From f3a17288736d9a685c266cc2e2481766fac8eb5f Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Wed, 22 Apr 2020 10:49:17 +0200
Subject: [PATCH 073/186] Tracking: signal key as modified when evicting.

---
 src/evict.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/evict.c b/src/evict.c
index 71260c040..a829f1f99 100644
--- a/src/evict.c
+++ b/src/evict.c
@@ -569,6 +569,7 @@ int freeMemoryIfNeeded(void) {
                 dbAsyncDelete(db,keyobj);
             else
                 dbSyncDelete(db,keyobj);
+            signalModifiedKey(NULL,db,keyobj);
             latencyEndMonitor(eviction_latency);
             latencyAddSampleIfNeeded("eviction-del",eviction_latency);
             latencyRemoveNestedEvent(latency,eviction_latency);

From e434b2ce4f4152141b7d03b585acf7171ab7161c Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Wed, 22 Apr 2020 11:24:19 +0200
Subject: [PATCH 074/186] Tracking: NOLOOP tests.

---
 tests/unit/tracking.tcl | 32 ++++++++++++++++++++++++++++++++
 1 file changed, 32 insertions(+)

diff --git a/tests/unit/tracking.tcl b/tests/unit/tracking.tcl
index 2058319f7..dc6cd671a 100644
--- a/tests/unit/tracking.tcl
+++ b/tests/unit/tracking.tcl
@@ -7,6 +7,9 @@ start_server {tags {"tracking"}} {
     $rd1 subscribe __redis__:invalidate
     $rd1 read ; # Consume the SUBSCRIBE reply.
 
+    # Create another client as well in order to test NOLOOP
+    set rd2 [redis_deferring_client]
+
     test {Clients are able to enable tracking and redirect it} {
         r CLIENT TRACKING on REDIRECT $redir
     } {*OK}
@@ -62,5 +65,34 @@ start_server {tags {"tracking"}} {
         assert {$keys eq {c:1234}}
     }
 
+    test {Tracking NOLOOP mode in standard mode works} {
+        r CLIENT TRACKING off
+        r CLIENT TRACKING on REDIRECT $redir NOLOOP
+        r MGET otherkey1 loopkey otherkey2
+        $rd2 SET otherkey1 1; # We should get this
+        r SET loopkey 1 ; # We should not get this
+        $rd2 SET otherkey2 1; # We should get this
+        # Because of the internals, we know we are going to receive
+        # two separated notifications for the two different prefixes.
+        set keys1 [lsort [lindex [$rd1 read] 2]]
+        set keys2 [lsort [lindex [$rd1 read] 2]]
+        set keys [lsort [list {*}$keys1 {*}$keys2]]
+        assert {$keys eq {otherkey1 otherkey2}}
+    }
+
+    test {Tracking NOLOOP mode in BCAST mode works} {
+        r CLIENT TRACKING off
+        r CLIENT TRACKING on BCAST REDIRECT $redir NOLOOP
+        $rd2 SET otherkey1 1; # We should get this
+        r SET loopkey 1 ; # We should not get this
+        $rd2 SET otherkey2 1; # We should get this
+        # Because of the internals, we know we are going to receive
+        # two separated notifications for the two different prefixes.
+        set keys1 [lsort [lindex [$rd1 read] 2]]
+        set keys2 [lsort [lindex [$rd1 read] 2]]
+        set keys [lsort [list {*}$keys1 {*}$keys2]]
+        assert {$keys eq {otherkey1 otherkey2}}
+    }
+
     $rd1 close
 }

From 06917e581ca2236f5e7ba2bc7cc0310c420bd1be Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Wed, 22 Apr 2020 11:45:34 +0200
Subject: [PATCH 075/186] Tracking: test expired keys notifications.

---
 tests/unit/tracking.tcl | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/tests/unit/tracking.tcl b/tests/unit/tracking.tcl
index dc6cd671a..43bb5f864 100644
--- a/tests/unit/tracking.tcl
+++ b/tests/unit/tracking.tcl
@@ -94,5 +94,18 @@ start_server {tags {"tracking"}} {
         assert {$keys eq {otherkey1 otherkey2}}
     }
 
+    test {Tracking gets notification of expired keys} {
+        r CLIENT TRACKING off
+        r CLIENT TRACKING on BCAST REDIRECT $redir NOLOOP
+        r SET mykey myval px 1
+        r SET mykeyotherkey myval ; # We should not get it
+        after 1000
+        # Because of the internals, we know we are going to receive
+        # two separated notifications for the two different prefixes.
+        set keys1 [lsort [lindex [$rd1 read] 2]]
+        set keys [lsort [list {*}$keys1]]
+        assert {$keys eq {mykey}}
+    }
+
     $rd1 close
 }

From 149b658b56503488705fec278f01864878540d76 Mon Sep 17 00:00:00 2001
From: Yossi Gottlieb <yossigo@gmail.com>
Date: Mon, 20 Apr 2020 13:34:37 +0300
Subject: [PATCH 076/186] TLS: Fix build on older verisons of OpenSSL.

---
 src/tls.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/tls.c b/src/tls.c
index 5fac6902b..ea0c34469 100644
--- a/src/tls.c
+++ b/src/tls.c
@@ -168,7 +168,9 @@ int tlsConfigure(redisTLSContextConfig *ctx_config) {
 
     SSL_CTX_set_mode(ctx, SSL_MODE_ENABLE_PARTIAL_WRITE|SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER);
     SSL_CTX_set_verify(ctx, SSL_VERIFY_PEER|SSL_VERIFY_FAIL_IF_NO_PEER_CERT, NULL);
+#if defined(SSL_CTX_set_ecdh_auto)
     SSL_CTX_set_ecdh_auto(ctx, 1);
+#endif
 
     if (SSL_CTX_use_certificate_file(ctx, ctx_config->cert_file, SSL_FILETYPE_PEM) <= 0) {
         ERR_error_string_n(ERR_get_error(), errbuf, sizeof(errbuf));

From b0920e6e861acd05370f8c80688615b158c46b38 Mon Sep 17 00:00:00 2001
From: Theo Buehler <theo.buehler@cysec.systems>
Date: Wed, 22 Apr 2020 09:43:01 +0200
Subject: [PATCH 077/186] TLS: Fix build with SSL_OP_NO_CLIENT_RENEGOTIATION

There is no ssl in this scope, so the build breaks.
All the other options are set directly on the ctx.
---
 src/tls.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/tls.c b/src/tls.c
index ea0c34469..c18aafebe 100644
--- a/src/tls.c
+++ b/src/tls.c
@@ -160,7 +160,7 @@ int tlsConfigure(redisTLSContextConfig *ctx_config) {
 #endif
 
 #ifdef SSL_OP_NO_CLIENT_RENEGOTIATION
-    SSL_CTX_set_options(ssl->ctx, SSL_OP_NO_CLIENT_RENEGOTIATION);
+    SSL_CTX_set_options(ctx, SSL_OP_NO_CLIENT_RENEGOTIATION);
 #endif
 
     if (ctx_config->prefer_server_ciphers)

From 321acea038a32b95cce3b379ff189cc832bed08a Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Wed, 22 Apr 2020 17:14:15 +0200
Subject: [PATCH 078/186] ACL: deny commands execution of disabled users.

---
 src/acl.c | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/acl.c b/src/acl.c
index 75b954c5e..9e2ed6af7 100644
--- a/src/acl.c
+++ b/src/acl.c
@@ -1055,6 +1055,10 @@ int ACLCheckCommandPerm(client *c, int *keyidxptr) {
     /* If there is no associated user, the connection can run anything. */
     if (u == NULL) return ACL_OK;
 
+    /* If the user is disabled we don't allow the execution of any
+     * command. */
+    if (!(u->flags & USER_FLAG_ENABLED)) return ACL_DENIED_CMD;
+
     /* Check if the user can execute this command. */
     if (!(u->flags & USER_FLAG_ALLCOMMANDS) &&
         c->cmd->proc != authCommand)

From 639c8a1d915092b198cb8ad89fc314a85e20649f Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Thu, 23 Apr 2020 10:39:53 +0200
Subject: [PATCH 079/186] ACL GENPASS: emit 256 bits instead of 128.

---
 src/acl.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/acl.c b/src/acl.c
index 9e2ed6af7..228811cba 100644
--- a/src/acl.c
+++ b/src/acl.c
@@ -1819,7 +1819,7 @@ void aclCommand(client *c) {
         dictReleaseIterator(di);
         setDeferredArrayLen(c,dl,arraylen);
     } else if (!strcasecmp(sub,"genpass") && c->argc == 2) {
-        char pass[32]; /* 128 bits of actual pseudo random data. */
+        char pass[64]; /* 256 bits of actual pseudo random data. */
         getRandomHexChars(pass,sizeof(pass));
         addReplyBulkCBuffer(c,pass,sizeof(pass));
     } else if (!strcasecmp(sub,"log") && (c->argc == 2 || c->argc ==3)) {

From 345c3768ded48383147e1196bf3bd18ca423f05b Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Thu, 23 Apr 2020 10:53:21 +0200
Subject: [PATCH 080/186] ACL GENPASS: take number of bits as argument.

---
 src/acl.c | 27 +++++++++++++++++++++------
 1 file changed, 21 insertions(+), 6 deletions(-)

diff --git a/src/acl.c b/src/acl.c
index 228811cba..bf5dd18f1 100644
--- a/src/acl.c
+++ b/src/acl.c
@@ -1626,7 +1626,7 @@ void addACLLogEntry(client *c, int reason, int keypos, sds username) {
  * ACL SETUSER <username> ... acl rules ...
  * ACL DELUSER <username> [...]
  * ACL GETUSER <username>
- * ACL GENPASS
+ * ACL GENPASS [<bits>]
  * ACL WHOAMI
  * ACL LOG [<count> | RESET]
  */
@@ -1818,10 +1818,25 @@ void aclCommand(client *c) {
         }
         dictReleaseIterator(di);
         setDeferredArrayLen(c,dl,arraylen);
-    } else if (!strcasecmp(sub,"genpass") && c->argc == 2) {
-        char pass[64]; /* 256 bits of actual pseudo random data. */
-        getRandomHexChars(pass,sizeof(pass));
-        addReplyBulkCBuffer(c,pass,sizeof(pass));
+    } else if (!strcasecmp(sub,"genpass") && (c->argc == 2 || c->argc == 3)) {
+        #define GENPASS_MAX_BITS 4096
+        char pass[GENPASS_MAX_BITS/8*2]; /* Hex representation. */
+        long bits = 256; /* By default generate 256 bits passwords. */
+
+        if (c->argc == 3 && getLongFromObjectOrReply(c,c->argv[2],&bits,NULL)
+            != C_OK) return;
+
+        if (bits <= 0 || bits > GENPASS_MAX_BITS) {
+            addReplyErrorFormat(c,
+                "ACL GENPASS argument must be the number of "
+                "bits for the output password, a positive number "
+                "up to %d",GENPASS_MAX_BITS);
+            return;
+        }
+
+        long chars = (bits+3)/4; /* Round to number of characters to emit. */
+        getRandomHexChars(pass,chars);
+        addReplyBulkCBuffer(c,pass,chars);
     } else if (!strcasecmp(sub,"log") && (c->argc == 2 || c->argc ==3)) {
         long count = 10; /* Number of entries to emit by default. */
 
@@ -1899,7 +1914,7 @@ void aclCommand(client *c) {
 "DELUSER <username> [...]         -- Delete a list of users.",
 "CAT                              -- List available categories.",
 "CAT <category>                   -- List commands inside category.",
-"GENPASS                          -- Generate a secure user password.",
+"GENPASS [<bits>]                 -- Generate a secure user password.",
 "WHOAMI                           -- Return the current connection username.",
 "LOG [<count> | RESET]            -- Show the ACL log entries.",
 NULL

From 05a41da75b0379580e808d11a946e954004ce3d4 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Thu, 23 Apr 2020 11:17:42 +0200
Subject: [PATCH 081/186] getRandomBytes(): use HMAC-SHA256.

Now that we have an interface to use this API directly, via ACL GENPASS,
we are no longer sure what people could do with it. So why don't make it
a strong primitive exported by Redis in order to create unique IDs and
so forth?

The implementation was tested against the test vectors that can
be found in RFC4231.
---
 src/util.c | 38 +++++++++++++++++++++++++++++---------
 1 file changed, 29 insertions(+), 9 deletions(-)

diff --git a/src/util.c b/src/util.c
index bd8f0fb98..d173f776f 100644
--- a/src/util.c
+++ b/src/util.c
@@ -42,7 +42,7 @@
 #include <time.h>
 
 #include "util.h"
-#include "sha1.h"
+#include "sha256.h"
 
 /* Glob-style pattern matching. */
 int stringmatchlen(const char *pattern, int patternLen,
@@ -622,7 +622,7 @@ int ld2string(char *buf, size_t len, long double value, ld2string_mode mode) {
 void getRandomBytes(unsigned char *p, size_t len) {
     /* Global state. */
     static int seed_initialized = 0;
-    static unsigned char seed[20]; /* The SHA1 seed, from /dev/urandom. */
+    static unsigned char seed[64]; /* 512 bit internal block size. */
     static uint64_t counter = 0; /* The counter we hash with the seed. */
 
     if (!seed_initialized) {
@@ -647,14 +647,34 @@ void getRandomBytes(unsigned char *p, size_t len) {
     }
 
     while(len) {
-        unsigned char digest[20];
-        SHA1_CTX ctx;
-        unsigned int copylen = len > 20 ? 20 : len;
+        /* This implements SHA256-HMAC. */
+        unsigned char digest[SHA256_BLOCK_SIZE];
+        unsigned char kxor[64];
+        unsigned int copylen =
+            len > SHA256_BLOCK_SIZE ? SHA256_BLOCK_SIZE : len;
 
-        SHA1Init(&ctx);
-        SHA1Update(&ctx, seed, sizeof(seed));
-        SHA1Update(&ctx, (unsigned char*)&counter,sizeof(counter));
-        SHA1Final(digest, &ctx);
+        /* IKEY: key xored with 0x36. */
+        memcpy(kxor,seed,sizeof(kxor));
+        for (unsigned int i = 0; i < sizeof(kxor); i++) kxor[i] ^= 0x36;
+
+        /* Obtain HASH(IKEY||MESSAGE). */
+        SHA256_CTX ctx;
+        sha256_init(&ctx);
+        sha256_update(&ctx,kxor,sizeof(kxor));
+        sha256_update(&ctx,(unsigned char*)&counter,sizeof(counter));
+        sha256_final(&ctx,digest);
+
+        /* OKEY: key xored with 0x5c. */
+        memcpy(kxor,seed,sizeof(kxor));
+        for (unsigned int i = 0; i < sizeof(kxor); i++) kxor[i] ^= 0x5C;
+
+        /* Obtain HASH(OKEY || HASH(IKEY||MESSAGE)). */
+        sha256_init(&ctx);
+        sha256_update(&ctx,kxor,sizeof(kxor));
+        sha256_update(&ctx,digest,SHA256_BLOCK_SIZE);
+        sha256_final(&ctx,digest);
+
+        /* Increment the counter for the next iteration. */
         counter++;
 
         memcpy(p,digest,copylen);

From fbdef6a9bd1ee6639124d6b949f484d2065e6c13 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Thu, 23 Apr 2020 11:56:36 +0200
Subject: [PATCH 082/186] ACL: re-enable command execution of disabled users.

After all I changed idea again: enabled/disabled should have a more
clear meaning, and it only means: you can't authenticate with such user
with new connections, however old connections continue to work as
expected.
---
 src/acl.c | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/src/acl.c b/src/acl.c
index bf5dd18f1..3194feb5b 100644
--- a/src/acl.c
+++ b/src/acl.c
@@ -1055,10 +1055,6 @@ int ACLCheckCommandPerm(client *c, int *keyidxptr) {
     /* If there is no associated user, the connection can run anything. */
     if (u == NULL) return ACL_OK;
 
-    /* If the user is disabled we don't allow the execution of any
-     * command. */
-    if (!(u->flags & USER_FLAG_ENABLED)) return ACL_DENIED_CMD;
-
     /* Check if the user can execute this command. */
     if (!(u->flags & USER_FLAG_ALLCOMMANDS) &&
         c->cmd->proc != authCommand)

From f0a261448c3832b9dffd55b89e5952cb6e47d279 Mon Sep 17 00:00:00 2001
From: Valentino Geron <valentino@redislabs.com>
Date: Tue, 21 Apr 2020 20:55:43 +0300
Subject: [PATCH 083/186] XREADGROUP with NOACK should propagate only one
 XGROUP SETID command

---
 src/t_stream.c | 20 +++++++++++++-------
 1 file changed, 13 insertions(+), 7 deletions(-)

diff --git a/src/t_stream.c b/src/t_stream.c
index 3efaa4509..8abbee02f 100644
--- a/src/t_stream.c
+++ b/src/t_stream.c
@@ -935,6 +935,8 @@ size_t streamReplyWithRange(client *c, stream *s, streamID *start, streamID *end
     streamIterator si;
     int64_t numfields;
     streamID id;
+    int propagate_last_id = 0;
+    int noack = flags & STREAM_RWR_NOACK;
 
     /* If the client is asking for some history, we serve it using a
      * different function, so that we return entries *solely* from its
@@ -950,12 +952,14 @@ size_t streamReplyWithRange(client *c, stream *s, streamID *start, streamID *end
         arraylen_ptr = addReplyDeferredLen(c);
     streamIteratorStart(&si,s,start,end,rev);
     while(streamIteratorGetID(&si,&id,&numfields)) {
-        int propagate_last_id = 0;
-
         /* Update the group last_id if needed. */
         if (group && streamCompareID(&id,&group->last_id) > 0) {
             group->last_id = id;
-            propagate_last_id = 1;
+            /* Group last id should be propagated only if NOACK was
+             * specified, otherwise the last id would be included
+             * in XCLAIM. */
+            if (noack)
+                propagate_last_id = 1;
         }
 
         /* Emit a two elements array for each item. The first is
@@ -983,7 +987,7 @@ size_t streamReplyWithRange(client *c, stream *s, streamID *start, streamID *end
          * XGROUP SETID command. So if we find that there is already
          * a NACK for the entry, we need to associate it to the new
          * consumer. */
-        if (group && !(flags & STREAM_RWR_NOACK)) {
+        if (group && !noack) {
             unsigned char buf[sizeof(streamID)];
             streamEncodeID(buf,&id);
 
@@ -1020,14 +1024,16 @@ size_t streamReplyWithRange(client *c, stream *s, streamID *start, streamID *end
                 streamPropagateXCLAIM(c,spi->keyname,group,spi->groupname,idarg,nack);
                 decrRefCount(idarg);
             }
-        } else {
-            if (propagate_last_id)
-                streamPropagateGroupID(c,spi->keyname,group,spi->groupname);
         }
 
         arraylen++;
         if (count && count == arraylen) break;
     }
+
+    if (spi && propagate_last_id) {
+        streamPropagateGroupID(c,spi->keyname,group,spi->groupname);
+    }
+
     streamIteratorStop(&si);
     if (arraylen_ptr) setDeferredArrayLen(c,arraylen_ptr,arraylen);
     return arraylen;

From 4db38d2efdb0b606dc1f6e3feeb12c97ab6bb554 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Thu, 23 Apr 2020 16:13:45 +0200
Subject: [PATCH 084/186] Minor aesthetic changes to #7135.

---
 src/t_stream.c | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)

diff --git a/src/t_stream.c b/src/t_stream.c
index 8abbee02f..596d8ad0d 100644
--- a/src/t_stream.c
+++ b/src/t_stream.c
@@ -955,11 +955,10 @@ size_t streamReplyWithRange(client *c, stream *s, streamID *start, streamID *end
         /* Update the group last_id if needed. */
         if (group && streamCompareID(&id,&group->last_id) > 0) {
             group->last_id = id;
-            /* Group last id should be propagated only if NOACK was
-             * specified, otherwise the last id would be included
-             * in XCLAIM. */
-            if (noack)
-                propagate_last_id = 1;
+            /* Group last ID should be propagated only if NOACK was
+             * specified, otherwise the last id will be included
+             * in the propagation of XCLAIM itself. */
+            if (noack) propagate_last_id = 1;
         }
 
         /* Emit a two elements array for each item. The first is
@@ -1030,9 +1029,8 @@ size_t streamReplyWithRange(client *c, stream *s, streamID *start, streamID *end
         if (count && count == arraylen) break;
     }
 
-    if (spi && propagate_last_id) {
+    if (spi && propagate_last_id)
         streamPropagateGroupID(c,spi->keyname,group,spi->groupname);
-    }
 
     streamIteratorStop(&si);
     if (arraylen_ptr) setDeferredArrayLen(c,arraylen_ptr,arraylen);

From 374ffdf1c1a057517e9769522d0973ad52f3a8b6 Mon Sep 17 00:00:00 2001
From: yanhui13 <yanhui13@meituan.com>
Date: Tue, 21 Apr 2020 16:55:05 +0800
Subject: [PATCH 085/186] optimize the output of cluster slots

---
 src/cluster.c | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/src/cluster.c b/src/cluster.c
index b103e2fe1..e3adaf83a 100644
--- a/src/cluster.c
+++ b/src/cluster.c
@@ -4191,11 +4191,17 @@ void clusterReplyMultiBulkSlots(client *c) {
     while((de = dictNext(di)) != NULL) {
         clusterNode *node = dictGetVal(de);
         int j = 0, start = -1;
+        int i, nested_elements = 0;
 
         /* Skip slaves (that are iterated when producing the output of their
          * master) and  masters not serving any slot. */
         if (!nodeIsMaster(node) || node->numslots == 0) continue;
 
+        for(i = 0; i < node->numslaves; i++) {
+            if (nodeFailed(node->slaves[i])) continue;
+            nested_elements++;
+        }
+
         for (j = 0; j < CLUSTER_SLOTS; j++) {
             int bit, i;
 
@@ -4203,8 +4209,7 @@ void clusterReplyMultiBulkSlots(client *c) {
                 if (start == -1) start = j;
             }
             if (start != -1 && (!bit || j == CLUSTER_SLOTS-1)) {
-                int nested_elements = 3; /* slots (2) + master addr (1). */
-                void *nested_replylen = addReplyDeferredLen(c);
+                addReplyArrayLen(c, nested_elements + 3); /* slots (2) + master addr (1). */
 
                 if (bit && j == CLUSTER_SLOTS-1) j++;
 
@@ -4234,9 +4239,7 @@ void clusterReplyMultiBulkSlots(client *c) {
                     addReplyBulkCString(c, node->slaves[i]->ip);
                     addReplyLongLong(c, node->slaves[i]->port);
                     addReplyBulkCBuffer(c, node->slaves[i]->name, CLUSTER_NAMELEN);
-                    nested_elements++;
                 }
-                setDeferredArrayLen(c, nested_replylen, nested_elements);
                 num_masters++;
             }
         }

From f03f1fad67b203d029e7b2bb8dda3c70a86fd47e Mon Sep 17 00:00:00 2001
From: yanhui13 <yanhui13@meituan.com>
Date: Tue, 21 Apr 2020 16:56:10 +0800
Subject: [PATCH 086/186] add tcl test for cluster slots

---
 tests/cluster/tests/15-cluster-slots.tcl | 44 ++++++++++++++++++++++++
 1 file changed, 44 insertions(+)
 create mode 100644 tests/cluster/tests/15-cluster-slots.tcl

diff --git a/tests/cluster/tests/15-cluster-slots.tcl b/tests/cluster/tests/15-cluster-slots.tcl
new file mode 100644
index 000000000..dc9938ef6
--- /dev/null
+++ b/tests/cluster/tests/15-cluster-slots.tcl
@@ -0,0 +1,44 @@
+source "../tests/includes/init-tests.tcl"
+
+proc cluster_allocate_mixedSlots {n} {
+    set slot 16383
+    while {$slot >= 0} {
+        set node [expr {$slot % $n}]
+        lappend slots_$node $slot
+        incr slot -1
+    }
+    for {set j 0} {$j < $n} {incr j} {
+        R $j cluster addslots {*}[set slots_${j}]
+    }
+}
+
+proc create_cluster_with_mixedSlot {masters slaves} {
+    cluster_allocate_mixedSlots $masters
+    if {$slaves} {
+        cluster_allocate_slaves $masters $slaves
+    }
+    assert_cluster_state ok
+}
+
+test "Create a 5 nodes cluster" {
+    create_cluster_with_mixedSlot 5 15
+}
+
+test "Cluster is up" {
+    assert_cluster_state ok
+}
+
+test "Cluster is writable" {
+    cluster_write_test 0
+}
+
+test "Instance #5 is a slave" {
+    assert {[RI 5 role] eq {slave}}
+}
+
+test "client do not break when cluster slot" {
+    R 0 config set client-output-buffer-limit "normal 33554432 16777216 60"
+    if { [catch {R 0 cluster slots}] } {
+        fail "output overflow when cluster slots"
+    }
+}
\ No newline at end of file

From 373ae6061afcc189ee642ac441c72f16621720ca Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Fri, 24 Apr 2020 10:13:20 +0200
Subject: [PATCH 087/186] Also use propagate() in streamPropagateGroupID().

---
 src/t_stream.c | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/src/t_stream.c b/src/t_stream.c
index 596d8ad0d..758db637e 100644
--- a/src/t_stream.c
+++ b/src/t_stream.c
@@ -848,6 +848,11 @@ void streamPropagateXCLAIM(client *c, robj *key, streamCG *group, robj *groupnam
     argv[11] = createStringObject("JUSTID",6);
     argv[12] = createStringObject("LASTID",6);
     argv[13] = createObjectFromStreamID(&group->last_id);
+
+    /* We use progagate() because this code path is not always called from
+     * the command execution context. Moreover this will just alter the
+     * consumer group state, and we don't need MULTI/EXEC wrapping because
+     * there is no message state cross-message atomicity required. */
     propagate(server.xclaimCommand,c->db->id,argv,14,PROPAGATE_AOF|PROPAGATE_REPL);
     decrRefCount(argv[0]);
     decrRefCount(argv[3]);
@@ -875,7 +880,12 @@ void streamPropagateGroupID(client *c, robj *key, streamCG *group, robj *groupna
     argv[2] = key;
     argv[3] = groupname;
     argv[4] = createObjectFromStreamID(&group->last_id);
-    alsoPropagate(server.xgroupCommand,c->db->id,argv,5,PROPAGATE_AOF|PROPAGATE_REPL);
+
+    /* We use progagate() because this code path is not always called from
+     * the command execution context. Moreover this will just alter the
+     * consumer group state, and we don't need MULTI/EXEC wrapping because
+     * there is no message state cross-message atomicity required. */
+    propagate(server.xgroupCommand,c->db->id,argv,5,PROPAGATE_AOF|PROPAGATE_REPL);
     decrRefCount(argv[0]);
     decrRefCount(argv[1]);
     decrRefCount(argv[4]);

From 3722f89f4936cbdbb2f456d7c55af42a016ad273 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Fri, 24 Apr 2020 16:49:27 +0200
Subject: [PATCH 088/186] LCS -> STRALGO LCS.

STRALGO should be a container for mostly read-only string
algorithms in Redis. The algorithms should have two main
characteristics:

1. They should be non trivial to compute, and often not part of
programming language standard libraries.
2. They should be fast enough that it is a good idea to have optimized C
implementations.

Next thing I would love to see? A small strings compression algorithm.
---
 src/server.c               |  2 +-
 src/server.h               |  2 +-
 src/t_string.c             | 23 ++++++++++++++++++-----
 tests/unit/type/string.tcl | 16 ++++++++--------
 4 files changed, 28 insertions(+), 15 deletions(-)

diff --git a/src/server.c b/src/server.c
index fc9b87aae..f7af79c3f 100644
--- a/src/server.c
+++ b/src/server.c
@@ -1006,7 +1006,7 @@ struct redisCommand redisCommandTable[] = {
      "admin no-script no-slowlog ok-loading ok-stale",
      0,NULL,0,0,0,0,0,0},
 
-    {"lcs",lcsCommand,-4,
+    {"stralgo",stralgoCommand,-2,
      "write use-memory @string",
      0,lcsGetKeys,0,0,0,0,0,0}
 };
diff --git a/src/server.h b/src/server.h
index d39359dce..9e1e506af 100644
--- a/src/server.h
+++ b/src/server.h
@@ -2389,7 +2389,7 @@ void xdelCommand(client *c);
 void xtrimCommand(client *c);
 void lolwutCommand(client *c);
 void aclCommand(client *c);
-void lcsCommand(client *c);
+void stralgoCommand(client *c);
 
 #if defined(__GNUC__)
 void *calloc(size_t count, size_t size) __attribute__ ((deprecated));
diff --git a/src/t_string.c b/src/t_string.c
index ef382bb0c..d4eb04769 100644
--- a/src/t_string.c
+++ b/src/t_string.c
@@ -480,18 +480,31 @@ void strlenCommand(client *c) {
     addReplyLongLong(c,stringObjectLen(o));
 }
 
-/* LCS -- Longest common subsequence.
+
+/* STRALGO -- Implement complex algorithms on strings.
  *
- * LCS [IDX] [MINMATCHLEN <len>]
- *     STRINGS <string> <string> | KEYS <keya> <keyb> */
-void lcsCommand(client *c) {
+ * STRALGO <algorithm> ... arguments ... */
+void stralgoLCS(client *c);     /* This implements the LCS algorithm. */
+void stralgoCommand(client *c) {
+    /* Select the algorithm. */
+    if (!strcasecmp(c->argv[1]->ptr,"lcs")) {
+        stralgoLCS(c);
+    } else {
+        addReply(c,shared.syntaxerr);
+    }
+}
+
+/* STRALGO <algo> [IDX] [MINMATCHLEN <len>] [WITHMATCHLEN]
+ *     STRINGS <string> <string> | KEYS <keya> <keyb>
+ */
+void stralgoLCS(client *c) {
     uint32_t i, j;
     long long minmatchlen = 0;
     sds a = NULL, b = NULL;
     int getlen = 0, getidx = 0, withmatchlen = 0;
     robj *obja = NULL, *objb = NULL;
 
-    for (j = 1; j < (uint32_t)c->argc; j++) {
+    for (j = 2; j < (uint32_t)c->argc; j++) {
         char *opt = c->argv[j]->ptr;
         int moreargs = (c->argc-1) - j;
 
diff --git a/tests/unit/type/string.tcl b/tests/unit/type/string.tcl
index b9ef9de7a..8126cdee8 100644
--- a/tests/unit/type/string.tcl
+++ b/tests/unit/type/string.tcl
@@ -424,29 +424,29 @@ start_server {tags {"string"}} {
     set rna2 {ATTAAAGGTTTATACCTTCCCAGGTAACAAACCAACCAACTTTCGATCTCTTGTAGATCTGTTCTCTAAACGAACTTTAAAATCTGTGTGGCTGTCACTCGGCTGCATGCTTAGTGCACTCACGCAGTATAATTAATAACTAATTACTGTCGTTGACAGGACACGAGTAACTCGTCTATCTTCTGCAGGCTGCTTACGGTTTCGTCCGTGTTGCAGCCGATCATCAGCACATCTAGGTTT}
     set rnalcs {ACCTTCCCAGGTAACAAACCAACCAACTTTCGATCTCTTGTAGATCTGTTCTCTAAACGAACTTTAAAATCTGTGTGGCTGTCACTCGGCTGCATGCTTAGTGCACTCACGCAGTATAATTAATAACTAATTACTGTCGTTGACAGGACACGAGTAACTCGTCTATCTTCTGCAGGCTGCTTACGGTTTCGTCCGTGTTGCAGCCGATCATCAGCACATCTAGGTTT}
 
-    test {LCS string output with STRINGS option} {
-        r LCS STRINGS $rna1 $rna2
+    test {STRALGO LCS string output with STRINGS option} {
+        r STRALGO LCS STRINGS $rna1 $rna2
     } $rnalcs
 
-    test {LCS len} {
-        r LCS LEN STRINGS $rna1 $rna2
+    test {STRALGO LCS len} {
+        r STRALGO LCS LEN STRINGS $rna1 $rna2
     } [string length $rnalcs]
 
     test {LCS with KEYS option} {
         r set virus1 $rna1
         r set virus2 $rna2
-        r LCS KEYS virus1 virus2
+        r STRALGO LCS KEYS virus1 virus2
     } $rnalcs
 
     test {LCS indexes} {
-        dict get [r LCS IDX KEYS virus1 virus2] matches
+        dict get [r STRALGO LCS IDX KEYS virus1 virus2] matches
     } {{{238 238} {239 239}} {{236 236} {238 238}} {{229 230} {236 237}} {{224 224} {235 235}} {{1 222} {13 234}}}
 
     test {LCS indexes with match len} {
-        dict get [r LCS IDX KEYS virus1 virus2 WITHMATCHLEN] matches
+        dict get [r STRALGO LCS IDX KEYS virus1 virus2 WITHMATCHLEN] matches
     } {{{238 238} {239 239} 1} {{236 236} {238 238} 1} {{229 230} {236 237} 2} {{224 224} {235 235} 1} {{1 222} {13 234} 222}}
 
     test {LCS indexes with match len and minimum match len} {
-        dict get [r LCS IDX KEYS virus1 virus2 WITHMATCHLEN MINMATCHLEN 5] matches
+        dict get [r STRALGO LCS IDX KEYS virus1 virus2 WITHMATCHLEN MINMATCHLEN 5] matches
     } {{{1 222} {13 234} 222}}
 }

From 1e17d3de79f79460670e273b43a473d7927dcea5 Mon Sep 17 00:00:00 2001
From: Dave-in-lafayette <64047859+Dave-in-lafayette@users.noreply.github.com>
Date: Mon, 20 Apr 2020 16:38:06 -0700
Subject: [PATCH 089/186] fix for crash during panic before all threads are up

If there's a panic before all threads have been started (say, if file descriptor 0 is closed at exec), the panic response will crash here again.
---
 src/bio.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/bio.c b/src/bio.c
index 0662c8c4c..85f681185 100644
--- a/src/bio.c
+++ b/src/bio.c
@@ -266,7 +266,7 @@ void bioKillThreads(void) {
     int err, j;
 
     for (j = 0; j < BIO_NUM_OPS; j++) {
-        if (pthread_cancel(bio_threads[j]) == 0) {
+        if (bio_threads[j] && pthread_cancel(bio_threads[j]) == 0) {
             if ((err = pthread_join(bio_threads[j],NULL)) != 0) {
                 serverLog(LL_WARNING,
                     "Bio thread for job type #%d can be joined: %s",

From 43329c9b64cff243adfa7f18e7782c2bfd0fe6a9 Mon Sep 17 00:00:00 2001
From: Guy Benoish <guy.benoish@redislabs.com>
Date: Sun, 19 Apr 2020 15:59:58 +0300
Subject: [PATCH 090/186] Add the stream tag to XSETID tests

---
 tests/unit/type/stream.tcl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/unit/type/stream.tcl b/tests/unit/type/stream.tcl
index 5de9f0571..c2b524d7f 100644
--- a/tests/unit/type/stream.tcl
+++ b/tests/unit/type/stream.tcl
@@ -414,7 +414,7 @@ start_server {tags {"stream"} overrides {appendonly yes stream-node-max-entries
     }
 }
 
-start_server {tags {"xsetid"}} {
+start_server {tags {"stream xsetid"}} {
     test {XADD can CREATE an empty stream} {
         r XADD mystream MAXLEN 0 * a b
         assert {[dict get [r xinfo stream mystream] length] == 0}

From 2144047e14d7c26a47ebcfaa8e19db5caf88f7c4 Mon Sep 17 00:00:00 2001
From: Dave-in-lafayette <64047859+Dave-in-lafayette@users.noreply.github.com>
Date: Mon, 20 Apr 2020 16:34:36 -0700
Subject: [PATCH 091/186] fix for unintended crash during panic response

If redis crashes early, before lua is set up (like, if File Descriptor 0 is closed before exec), it will crash again trying to print memory statistics.
---
 src/server.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/server.c b/src/server.c
index f7af79c3f..f2fe54e1c 100644
--- a/src/server.c
+++ b/src/server.c
@@ -4037,7 +4037,7 @@ sds genRedisInfoString(const char *section) {
         size_t zmalloc_used = zmalloc_used_memory();
         size_t total_system_mem = server.system_memory_size;
         const char *evict_policy = evictPolicyToString();
-        long long memory_lua = (long long)lua_gc(server.lua,LUA_GCCOUNT,0)*1024;
+        long long memory_lua = server.lua ? (long long)lua_gc(server.lua,LUA_GCCOUNT,0)*1024 : 0;
         struct redisMemOverhead *mh = getMemoryOverheadData();
 
         /* Peak memory is updated from time to time by serverCron() so it

From fea9788cc4ef02dee8b41a21d1f8ae56d79a86cb Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Mon, 27 Apr 2020 13:35:17 +0200
Subject: [PATCH 092/186] Fix STRALGO command flags.

---
 src/server.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/server.c b/src/server.c
index f2fe54e1c..db1d3780c 100644
--- a/src/server.c
+++ b/src/server.c
@@ -1007,7 +1007,7 @@ struct redisCommand redisCommandTable[] = {
      0,NULL,0,0,0,0,0,0},
 
     {"stralgo",stralgoCommand,-2,
-     "write use-memory @string",
+     "read-only @string",
      0,lcsGetKeys,0,0,0,0,0,0}
 };
 

From e4d2bb62b2922e549ae4c86f514a4c63aa7323e8 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Thu, 23 Apr 2020 15:04:42 +0300
Subject: [PATCH 093/186] Keep track of meaningful replication offset in
 replicas too

Now both master and replicas keep track of the last replication offset
that contains meaningful data (ignoring the tailing pings), and both
trim that tail from the replication backlog, and the offset with which
they try to use for psync.

the implication is that if someone missed some pings, or even have
excessive pings that the promoted replica has, it'll still be able to
psync (avoid full sync).

the downside (which was already committed) is that replicas running old
code may fail to psync, since the promoted replica trims pings form it's
backlog.

This commit adds a test that reproduces several cases of promotions and
demotions with stale and non-stale pings

Background:
The mearningful offset on the master was added recently to solve a problem were
the master is left all alone, injecting PINGs into it's backlog when no one is
listening and then gets demoted and tries to replicate from a replica that didn't
have any of the PINGs (or at least not the last ones).

however, consider this case:
master A has two replicas (B and C) replicating directly from it.
there's no traffic at all, and also no network issues, just many pings in the
tail of the backlog. now B gets promoted, A becomes a replica of B, and C
remains a replica of A. when A gets demoted, it trims the pings from its
backlog, and successfully replicate from B. however, C is still aware of
these PINGs, when it'll disconnect and re-connect to A, it'll ask for something
that's not in the backlog anymore (since A trimmed the tail of it's backlog),
and be forced to do a full sync (something it didn't have to do before the
meaningful offset fix).

Besides that, the psync2 test was always failing randomly here and there, it
turns out the reason were PINGs. Investigating it shows the following scenario:

cycle 1: redis #1 is master, and all the rest are direct replicas of #1
cycle 2: redis #2 is promoted to master, #1 is a replica of #2 and #3 is replica of #1
now we see that when #1 is demoted it prints:
17339:S 21 Apr 2020 11:16:38.523 * Using the meaningful offset 3929963 instead of 3929977 to exclude the final PINGs (14 bytes difference)
17339:S 21 Apr 2020 11:16:39.391 * Trying a partial resynchronization (request e2b3f8817735fdfe5fa4626766daa938b61419e5:3929964).
17339:S 21 Apr 2020 11:16:39.392 * Successful partial resynchronization with master.
and when #3 connects to the demoted #2, #2 says:
17339:S 21 Apr 2020 11:16:40.084 * Partial resynchronization not accepted: Requested offset for secondary ID was 3929978, but I can reply up to 3929964

so the issue here is that the meaningful offset feature saved the day for the
demoted master (since it needs to sync from a replica that didn't get the last
ping), but it didn't help one of the other replicas which did get the last ping.
---
 src/blocked.c                |   2 +-
 src/networking.c             |  93 +++++++++++-----------
 src/replication.c            |  66 +++++++++-------
 src/server.h                 |   1 -
 tests/integration/psync2.tcl | 144 ++++++++++++++++++++++++++++++-----
 5 files changed, 213 insertions(+), 93 deletions(-)

diff --git a/src/blocked.c b/src/blocked.c
index 00cc798d5..045369e93 100644
--- a/src/blocked.c
+++ b/src/blocked.c
@@ -110,7 +110,7 @@ void processUnblockedClients(void) {
          * the code is conceptually more correct this way. */
         if (!(c->flags & CLIENT_BLOCKED)) {
             if (c->querybuf && sdslen(c->querybuf) > 0) {
-                processInputBufferAndReplicate(c);
+                processInputBuffer(c);
             }
         }
     }
diff --git a/src/networking.c b/src/networking.c
index 744979d16..e4d40fdf0 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -1671,12 +1671,55 @@ int processMultibulkBuffer(client *c) {
     return C_ERR;
 }
 
+/* Perform necessary tasks after a command was executed:
+ *
+ * 1. The client is reset unless there are reasons to avoid doing it.
+ * 2. In the case of master clients, the replication offset is updated.
+ * 3. Propagate commands we got from our master to replicas down the line. */
+void commandProcessed(client *c) {
+    int cmd_is_ping = c->cmd && c->cmd->proc == pingCommand;
+    long long prev_offset = c->reploff;
+    if (c->flags & CLIENT_MASTER && !(c->flags & CLIENT_MULTI)) {
+        /* Update the applied replication offset of our master. */
+        c->reploff = c->read_reploff - sdslen(c->querybuf) + c->qb_pos;
+    }
+
+    /* Don't reset the client structure for clients blocked in a
+     * module blocking command, so that the reply callback will
+     * still be able to access the client argv and argc field.
+     * The client will be reset in unblockClientFromModule(). */
+    if (!(c->flags & CLIENT_BLOCKED) ||
+        c->btype != BLOCKED_MODULE)
+    {
+        resetClient(c);
+    }
+
+    /* If the client is a master we need to compute the difference
+     * between the applied offset before and after processing the buffer,
+     * to understand how much of the replication stream was actually
+     * applied to the master state: this quantity, and its corresponding
+     * part of the replication stream, will be propagated to the
+     * sub-replicas and to the replication backlog. */
+    if (c->flags & CLIENT_MASTER) {
+        long long applied = c->reploff - prev_offset;
+        long long prev_master_repl_meaningful_offset = server.master_repl_meaningful_offset;
+        if (applied) {
+            replicationFeedSlavesFromMasterStream(server.slaves,
+                    c->pending_querybuf, applied);
+            sdsrange(c->pending_querybuf,applied,-1);
+        }
+        /* The server.master_repl_meaningful_offset variable represents
+         * the offset of the replication stream without the pending PINGs. */
+        if (cmd_is_ping)
+            server.master_repl_meaningful_offset = prev_master_repl_meaningful_offset;
+    }
+}
+
 /* This function calls processCommand(), but also performs a few sub tasks
- * that are useful in that context:
+ * for the client that are useful in that context:
  *
  * 1. It sets the current client to the client 'c'.
- * 2. In the case of master clients, the replication offset is updated.
- * 3. The client is reset unless there are reasons to avoid doing it.
+ * 2. calls commandProcessed() if the command was handled.
  *
  * The function returns C_ERR in case the client was freed as a side effect
  * of processing the command, otherwise C_OK is returned. */
@@ -1684,20 +1727,7 @@ int processCommandAndResetClient(client *c) {
     int deadclient = 0;
     server.current_client = c;
     if (processCommand(c) == C_OK) {
-        if (c->flags & CLIENT_MASTER && !(c->flags & CLIENT_MULTI)) {
-            /* Update the applied replication offset of our master. */
-            c->reploff = c->read_reploff - sdslen(c->querybuf) + c->qb_pos;
-        }
-
-        /* Don't reset the client structure for clients blocked in a
-         * module blocking command, so that the reply callback will
-         * still be able to access the client argv and argc field.
-         * The client will be reset in unblockClientFromModule(). */
-        if (!(c->flags & CLIENT_BLOCKED) ||
-            c->btype != BLOCKED_MODULE)
-        {
-            resetClient(c);
-        }
+        commandProcessed(c);
     }
     if (server.current_client == NULL) deadclient = 1;
     server.current_client = NULL;
@@ -1794,31 +1824,6 @@ void processInputBuffer(client *c) {
     }
 }
 
-/* This is a wrapper for processInputBuffer that also cares about handling
- * the replication forwarding to the sub-replicas, in case the client 'c'
- * is flagged as master. Usually you want to call this instead of the
- * raw processInputBuffer(). */
-void processInputBufferAndReplicate(client *c) {
-    if (!(c->flags & CLIENT_MASTER)) {
-        processInputBuffer(c);
-    } else {
-        /* If the client is a master we need to compute the difference
-         * between the applied offset before and after processing the buffer,
-         * to understand how much of the replication stream was actually
-         * applied to the master state: this quantity, and its corresponding
-         * part of the replication stream, will be propagated to the
-         * sub-replicas and to the replication backlog. */
-        size_t prev_offset = c->reploff;
-        processInputBuffer(c);
-        size_t applied = c->reploff - prev_offset;
-        if (applied) {
-            replicationFeedSlavesFromMasterStream(server.slaves,
-                    c->pending_querybuf, applied);
-            sdsrange(c->pending_querybuf,applied,-1);
-        }
-    }
-}
-
 void readQueryFromClient(connection *conn) {
     client *c = connGetPrivateData(conn);
     int nread, readlen;
@@ -1886,7 +1891,7 @@ void readQueryFromClient(connection *conn) {
 
     /* There is more data in the client input buffer, continue parsing it
      * in case to check if there is a full command to execute. */
-     processInputBufferAndReplicate(c);
+     processInputBuffer(c);
 }
 
 void getClientsMaxBuffers(unsigned long *longest_output_list,
@@ -3101,7 +3106,7 @@ int handleClientsWithPendingReadsUsingThreads(void) {
                 continue;
             }
         }
-        processInputBufferAndReplicate(c);
+        processInputBuffer(c);
     }
     return processed;
 }
diff --git a/src/replication.c b/src/replication.c
index 3e9910374..c59639cd1 100644
--- a/src/replication.c
+++ b/src/replication.c
@@ -39,6 +39,7 @@
 #include <sys/socket.h>
 #include <sys/stat.h>
 
+long long adjustMeaningfulReplOffset();
 void replicationDiscardCachedMaster(void);
 void replicationResurrectCachedMaster(connection *conn);
 void replicationSendAck(void);
@@ -2693,6 +2694,9 @@ void replicationCacheMaster(client *c) {
      * pending outputs to the master. */
     sdsclear(server.master->querybuf);
     sdsclear(server.master->pending_querybuf);
+    /* Adjust reploff and read_reploff to the last meaningful offset we executed.
+     * this is the offset the replica will use for future PSYNC. */
+    server.master->reploff = adjustMeaningfulReplOffset();
     server.master->read_reploff = server.master->reploff;
     if (c->flags & CLIENT_MULTI) discardTransaction(c);
     listEmpty(c->reply);
@@ -2717,6 +2721,38 @@ void replicationCacheMaster(client *c) {
     replicationHandleMasterDisconnection();
 }
 
+/* If the "meaningful" offset, that is the offset without the final PINGs
+ * in the stream, is different than the last offset, use it instead:
+ * often when the master is no longer reachable, replicas will never
+ * receive the PINGs, however the master will end with an incremented
+ * offset because of the PINGs and will not be able to incrementally
+ * PSYNC with the new master.
+ * This function trims the replication backlog when needed, and returns
+ * the offset to be used for future partial sync. */
+long long adjustMeaningfulReplOffset() {
+    if (server.master_repl_offset > server.master_repl_meaningful_offset) {
+        long long delta = server.master_repl_offset -
+                          server.master_repl_meaningful_offset;
+        serverLog(LL_NOTICE,
+            "Using the meaningful offset %lld instead of %lld to exclude "
+            "the final PINGs (%lld bytes difference)",
+                server.master_repl_meaningful_offset,
+                server.master_repl_offset,
+                delta);
+        server.master_repl_offset = server.master_repl_meaningful_offset;
+        if (server.repl_backlog_histlen <= delta) {
+            server.repl_backlog_histlen = 0;
+            server.repl_backlog_idx = 0;
+        } else {
+            server.repl_backlog_histlen -= delta;
+            server.repl_backlog_idx =
+                (server.repl_backlog_idx + (server.repl_backlog_size - delta)) %
+                server.repl_backlog_size;
+        }
+    }
+    return server.master_repl_offset;
+}
+
 /* This function is called when a master is turend into a slave, in order to
  * create from scratch a cached master for the new client, that will allow
  * to PSYNC with the slave that was promoted as the new master after a
@@ -2736,35 +2772,7 @@ void replicationCacheMasterUsingMyself(void) {
      * by replicationCreateMasterClient(). We'll later set the created
      * master as server.cached_master, so the replica will use such
      * offset for PSYNC. */
-    server.master_initial_offset = server.master_repl_offset;
-
-    /* However if the "meaningful" offset, that is the offset without
-     * the final PINGs in the stream, is different, use this instead:
-     * often when the master is no longer reachable, replicas will never
-     * receive the PINGs, however the master will end with an incremented
-     * offset because of the PINGs and will not be able to incrementally
-     * PSYNC with the new master. */
-    if (server.master_repl_offset > server.master_repl_meaningful_offset) {
-        long long delta = server.master_repl_offset -
-                          server.master_repl_meaningful_offset;
-        serverLog(LL_NOTICE,
-            "Using the meaningful offset %lld instead of %lld to exclude "
-            "the final PINGs (%lld bytes difference)",
-                server.master_repl_meaningful_offset,
-                server.master_repl_offset,
-                delta);
-        server.master_initial_offset = server.master_repl_meaningful_offset;
-        server.master_repl_offset = server.master_repl_meaningful_offset;
-        if (server.repl_backlog_histlen <= delta) {
-            server.repl_backlog_histlen = 0;
-            server.repl_backlog_idx = 0;
-        } else {
-            server.repl_backlog_histlen -= delta;
-            server.repl_backlog_idx =
-                (server.repl_backlog_idx + (server.repl_backlog_size - delta)) %
-                server.repl_backlog_size;
-        }
-    }
+    server.master_initial_offset = adjustMeaningfulReplOffset();
 
     /* The master client we create can be set to any DBID, because
      * the new master will start its replication stream with SELECT. */
diff --git a/src/server.h b/src/server.h
index 9e1e506af..41d767e13 100644
--- a/src/server.h
+++ b/src/server.h
@@ -1600,7 +1600,6 @@ void setDeferredSetLen(client *c, void *node, long length);
 void setDeferredAttributeLen(client *c, void *node, long length);
 void setDeferredPushLen(client *c, void *node, long length);
 void processInputBuffer(client *c);
-void processInputBufferAndReplicate(client *c);
 void processGopherRequest(client *c);
 void acceptHandler(aeEventLoop *el, int fd, void *privdata, int mask);
 void acceptTcpHandler(aeEventLoop *el, int fd, void *privdata, int mask);
diff --git a/tests/integration/psync2.tcl b/tests/integration/psync2.tcl
index 333736ffa..4e1189e0b 100644
--- a/tests/integration/psync2.tcl
+++ b/tests/integration/psync2.tcl
@@ -44,6 +44,7 @@ start_server {} {
         set used [list $master_id]
         test "PSYNC2: \[NEW LAYOUT\] Set #$master_id as master" {
             $R($master_id) slaveof no one
+            $R($master_id) config set repl-ping-replica-period 1 ;# increse the chance that random ping will cause issues
             if {$counter_value == 0} {
                 $R($master_id) set x $counter_value
             }
@@ -114,23 +115,20 @@ start_server {} {
             }
         }
 
-        # wait for all the slaves to be in sync with the master
-        set master_ofs [status $R($master_id) master_repl_offset]
+        # wait for all the slaves to be in sync with the master, due to pings, we have to re-sample the master constantly too
         wait_for_condition 500 100 {
-            $master_ofs == [status $R(0) master_repl_offset] &&
-            $master_ofs == [status $R(1) master_repl_offset] &&
-            $master_ofs == [status $R(2) master_repl_offset] &&
-            $master_ofs == [status $R(3) master_repl_offset] &&
-            $master_ofs == [status $R(4) master_repl_offset]
+            [status $R($master_id) master_repl_offset] == [status $R(0) master_repl_offset] &&
+            [status $R($master_id) master_repl_offset] == [status $R(1) master_repl_offset] &&
+            [status $R($master_id) master_repl_offset] == [status $R(2) master_repl_offset] &&
+            [status $R($master_id) master_repl_offset] == [status $R(3) master_repl_offset] &&
+            [status $R($master_id) master_repl_offset] == [status $R(4) master_repl_offset]
         } else {
-            if {$debug_msg} {
-                for {set j 0} {$j < 5} {incr j} {
-                    puts "$j: sync_full: [status $R($j) sync_full]"
-                    puts "$j: id1      : [status $R($j) master_replid]:[status $R($j) master_repl_offset]"
-                    puts "$j: id2      : [status $R($j) master_replid2]:[status $R($j) second_repl_offset]"
-                    puts "$j: backlog  : firstbyte=[status $R($j) repl_backlog_first_byte_offset] len=[status $R($j) repl_backlog_histlen]"
-                    puts "---"
-                }
+            for {set j 0} {$j < 5} {incr j} {
+                puts "$j: sync_full: [status $R($j) sync_full]"
+                puts "$j: id1      : [status $R($j) master_replid]:[status $R($j) master_repl_offset]"
+                puts "$j: id2      : [status $R($j) master_replid2]:[status $R($j) second_repl_offset]"
+                puts "$j: backlog  : firstbyte=[status $R($j) repl_backlog_first_byte_offset] len=[status $R($j) repl_backlog_histlen]"
+                puts "---"
             }
             fail "Slaves are not in sync with the master after too long time."
         }
@@ -175,9 +173,14 @@ start_server {} {
             $R($j) slaveof $master_host $master_port
         }
 
-        # Wait for slaves to sync
+        # Wait for replicas to sync. it is not enough to just wait for connected_slaves==4
+        # since we might do the check before the master realized that they're disconnected
         wait_for_condition 50 1000 {
-            [status $R($master_id) connected_slaves] == 4
+            [status $R($master_id) connected_slaves] == 4 &&
+            [status $R([expr {($master_id+1)%5}]) master_link_status] == "up" &&
+            [status $R([expr {($master_id+2)%5}]) master_link_status] == "up" &&
+            [status $R([expr {($master_id+3)%5}]) master_link_status] == "up" &&
+            [status $R([expr {($master_id+4)%5}]) master_link_status] == "up"
         } else {
             fail "Replica not reconnecting"
         }
@@ -188,6 +191,7 @@ start_server {} {
         set slave_id [expr {($master_id+1)%5}]
         set sync_count [status $R($master_id) sync_full]
         set sync_partial [status $R($master_id) sync_partial_ok]
+        set sync_partial_err [status $R($master_id) sync_partial_err]
         catch {
             $R($slave_id) config rewrite
             $R($slave_id) debug restart
@@ -197,7 +201,11 @@ start_server {} {
         wait_for_condition 50 1000 {
             [status $R($master_id) sync_partial_ok] == $sync_partial + 1
         } else {
-            fail "Replica not reconnecting"
+            puts "prev sync_full: $sync_count"
+            puts "prev sync_partial_ok: $sync_partial"
+            puts "prev sync_partial_err: $sync_partial_err"
+            puts [$R($master_id) info stats]
+            fail "Replica didn't partial sync"
         }
         set new_sync_count [status $R($master_id) sync_full]
         assert {$sync_count == $new_sync_count}
@@ -271,3 +279,103 @@ start_server {} {
     }
 
 }}}}}
+
+start_server {tags {"psync2"}} {
+start_server {} {
+start_server {} {
+start_server {} {
+start_server {} {
+    test {pings at the end of replication stream are ignored for psync} {
+        set master [srv -4 client]
+        set master_host [srv -4 host]
+        set master_port [srv -4 port]
+        set replica1 [srv -3 client]
+        set replica2 [srv -2 client]
+        set replica3 [srv -1 client]
+        set replica4 [srv -0 client]
+
+        $replica1 replicaof $master_host $master_port
+        $replica2 replicaof $master_host $master_port
+        $replica3 replicaof $master_host $master_port
+        $replica4 replicaof $master_host $master_port
+        wait_for_condition 50 1000 {
+            [status $master connected_slaves] == 4
+        } else {
+            fail "replicas didn't connect"
+        }
+
+        $master incr x
+        wait_for_condition 50 1000 {
+            [$replica1 get x] == 1 && [$replica2 get x] == 1 &&
+            [$replica3 get x] == 1 && [$replica4 get x] == 1
+        } else {
+            fail "replicas didn't get incr"
+        }
+
+        # disconnect replica1 and replica2
+        # and wait for the master to send a ping to replica3 and replica4
+        $replica1 replicaof no one
+        $replica2 replicaof 127.0.0.1 1 ;# we can't promote it to master since that will cycle the replication id
+        $master config set repl-ping-replica-period 1
+        after 1500
+
+        # make everyone sync from the replica1 that didn't get the last ping from the old master
+        # replica4 will keep syncing from the old master which now syncs from replica1
+        # and replica2 will re-connect to the old master (which went back in time)
+        set new_master_host [srv -3 host]
+        set new_master_port [srv -3 port]
+        $replica3 replicaof $new_master_host $new_master_port
+        $master replicaof $new_master_host $new_master_port
+        $replica2 replicaof $master_host $master_port
+        wait_for_condition 50 1000 {
+            [status $replica2 master_link_status] == "up" &&
+            [status $replica3 master_link_status] == "up" &&
+            [status $replica4 master_link_status] == "up" &&
+            [status $master master_link_status] == "up"
+        } else {
+            fail "replicas didn't connect"
+        }
+
+        # make sure replication is still alive and kicking
+        $replica1 incr x
+        wait_for_condition 50 1000 {
+            [$replica2 get x] == 2 &&
+            [$replica3 get x] == 2 &&
+            [$replica4 get x] == 2 &&
+            [$master get x] == 2
+        } else {
+            fail "replicas didn't get incr"
+        }
+
+        # make sure there are full syncs other than the initial ones
+        assert_equal [status $master sync_full] 4
+        assert_equal [status $replica1 sync_full] 0
+        assert_equal [status $replica2 sync_full] 0
+        assert_equal [status $replica3 sync_full] 0
+        assert_equal [status $replica4 sync_full] 0
+
+        # force psync
+        $master client kill type master
+        $replica2 client kill type master
+        $replica3 client kill type master
+        $replica4 client kill type master
+
+        # make sure replication is still alive and kicking
+        $replica1 incr x
+        wait_for_condition 50 1000 {
+            [$replica2 get x] == 3 &&
+            [$replica3 get x] == 3 &&
+            [$replica4 get x] == 3 &&
+            [$master get x] == 3
+        } else {
+            fail "replicas didn't get incr"
+        }
+
+        # make sure there are full syncs other than the initial ones
+        assert_equal [status $master sync_full] 4
+        assert_equal [status $replica1 sync_full] 0
+        assert_equal [status $replica2 sync_full] 0
+        assert_equal [status $replica3 sync_full] 0
+        assert_equal [status $replica4 sync_full] 0
+}
+}}}}}

From 8110ba888072c9f5b3eba7965f6f3dcc20bcd628 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Fri, 24 Apr 2020 17:20:28 +0300
Subject: [PATCH 094/186] optimize memory usage of deferred replies

When deffered reply is added the previous reply node cannot be used so
all the extra space we allocated in it is wasted. in case someone uses
deffered replies in a loop, each time adding a small reply, each of
these reply nodes (the small string reply) would have consumed a 16k
block.
now when we add anther diferred reply node, we trim the unused portion
of the previous reply block.

see #7123
---
 src/networking.c | 31 +++++++++++++++++++++++++++++++
 1 file changed, 31 insertions(+)

diff --git a/src/networking.c b/src/networking.c
index e4d40fdf0..7ffa99eb1 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -436,6 +436,36 @@ void addReplyStatusFormat(client *c, const char *fmt, ...) {
     sdsfree(s);
 }
 
+/* Sometimes we are forced to create a new reply node, and we can't append to
+ * the previous one, when that happens, we wanna try to trim the unused space
+ * at the end of the last reply node which we won't use anymore. */
+void trimReplyUnusedTailSpace(client *c) {
+    listNode *ln = listLast(c->reply);
+    clientReplyBlock *tail = ln? listNodeValue(ln): NULL;
+
+    /* Note that 'tail' may be NULL even if we have a tail node, becuase when
+     * addDeferredMultiBulkLength() is used */
+    if (!tail) return;
+
+    /* We only try to trim the space is relatively high (more than a 1/4 of the
+     * allocation), otherwise there's a high chance realloc will NOP.
+     * Also, to avoid large memmove which happens as part of realloc, we only do
+     * that if the used part is small.  */
+    if (tail->size - tail->used > tail->size / 4 &&
+        tail->used < PROTO_REPLY_CHUNK_BYTES)
+    {
+        size_t old_size = tail->size;
+        tail = zrealloc(tail, tail->used + sizeof(clientReplyBlock));
+        /* If realloc was a NOP, we got the same value which has internal frag */
+        if (tail == listNodeValue(ln)) return;
+        /* take over the allocation's internal fragmentation (at least for
+         * memory usage tracking) */
+        tail->size = zmalloc_usable(tail) - sizeof(clientReplyBlock);
+        c->reply_bytes += tail->size - old_size;
+        listNodeValue(ln) = tail;
+    }
+}
+
 /* Adds an empty object to the reply list that will contain the multi bulk
  * length, which is not known when this function is called. */
 void *addReplyDeferredLen(client *c) {
@@ -443,6 +473,7 @@ void *addReplyDeferredLen(client *c) {
      * ready to be sent, since we are sure that before returning to the
      * event loop setDeferredAggregateLen() will be called. */
     if (prepareClientToWrite(c) != C_OK) return NULL;
+    trimReplyUnusedTailSpace(c);
     listAddNodeTail(c->reply,NULL); /* NULL is our placeholder. */
     return listLast(c->reply);
 }

From 52c75e9db16745b9550cd08d0b62e241f1d9a8a3 Mon Sep 17 00:00:00 2001
From: Madelyn Olson <matolson@amazon.com>
Date: Fri, 24 Apr 2020 16:59:24 -0700
Subject: [PATCH 095/186] Implemented CRC64 based on slice by 4

---
 src/Makefile |   4 +-
 src/crc64.c  | 270 ++++++++++++++++++++++-----------------------------
 src/crc64.h  |   1 +
 src/rdb.c    |   3 +-
 src/server.c |   1 +
 5 files changed, 123 insertions(+), 156 deletions(-)

diff --git a/src/Makefile b/src/Makefile
index 3f982cc8e..f9922afce 100644
--- a/src/Makefile
+++ b/src/Makefile
@@ -206,9 +206,9 @@ endif
 
 REDIS_SERVER_NAME=redis-server
 REDIS_SENTINEL_NAME=redis-sentinel
-REDIS_SERVER_OBJ=adlist.o quicklist.o ae.o anet.o dict.o server.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o cluster.o crc16.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o memtest.o crc64.o bitops.o sentinel.o notify.o setproctitle.o blocked.o hyperloglog.o latency.o sparkline.o redis-check-rdb.o redis-check-aof.o geo.o lazyfree.o module.o evict.o expire.o geohash.o geohash_helper.o childinfo.o defrag.o siphash.o rax.o t_stream.o listpack.o localtime.o lolwut.o lolwut5.o lolwut6.o acl.o gopher.o tracking.o connection.o tls.o sha256.o timeout.o
+REDIS_SERVER_OBJ=adlist.o quicklist.o ae.o anet.o dict.o server.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o cluster.o crc16.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o memtest.o crcspeed.o crc64.o bitops.o sentinel.o notify.o setproctitle.o blocked.o hyperloglog.o latency.o sparkline.o redis-check-rdb.o redis-check-aof.o geo.o lazyfree.o module.o evict.o expire.o geohash.o geohash_helper.o childinfo.o defrag.o siphash.o rax.o t_stream.o listpack.o localtime.o lolwut.o lolwut5.o lolwut6.o acl.o gopher.o tracking.o connection.o tls.o sha256.o timeout.o
 REDIS_CLI_NAME=redis-cli
-REDIS_CLI_OBJ=anet.o adlist.o dict.o redis-cli.o zmalloc.o release.o ae.o crc64.o siphash.o crc16.o
+REDIS_CLI_OBJ=anet.o adlist.o dict.o redis-cli.o zmalloc.o release.o ae.o crcspeed.o crc64.o siphash.o crc16.o
 REDIS_BENCHMARK_NAME=redis-benchmark
 REDIS_BENCHMARK_OBJ=ae.o anet.o redis-benchmark.o adlist.o dict.o zmalloc.o siphash.o
 REDIS_CHECK_RDB_NAME=redis-check-rdb
diff --git a/src/crc64.c b/src/crc64.c
index f1f764922..165941dea 100644
--- a/src/crc64.c
+++ b/src/crc64.c
@@ -1,16 +1,5 @@
-/* Redis uses the CRC64 variant with "Jones" coefficients and init value of 0.
- *
- * Specification of this CRC64 variant follows:
- * Name: crc-64-jones
- * Width: 64 bites
- * Poly: 0xad93d23594c935a9
- * Reflected In: True
- * Xor_In: 0xffffffffffffffff
- * Reflected_Out: True
- * Xor_Out: 0x0
- * Check("123456789"): 0xe9c6d914c4b8d9ca
- *
- * Copyright (c) 2012, Salvatore Sanfilippo <antirez at gmail dot com>
+/* Copyright (c) 2014, Matt Stancliff <matt@genges.com>
+ * Copyright (c) 2020, Amazon Web Services
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -37,159 +26,134 @@
  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  * POSSIBILITY OF SUCH DAMAGE. */
 
-#include <stdint.h>
+#include "crc64.h"
+#include "crcspeed.h"
+static uint64_t crc64_table[8][256] = {{0}};
 
-static const uint64_t crc64_tab[256] = {
-    UINT64_C(0x0000000000000000), UINT64_C(0x7ad870c830358979),
-    UINT64_C(0xf5b0e190606b12f2), UINT64_C(0x8f689158505e9b8b),
-    UINT64_C(0xc038e5739841b68f), UINT64_C(0xbae095bba8743ff6),
-    UINT64_C(0x358804e3f82aa47d), UINT64_C(0x4f50742bc81f2d04),
-    UINT64_C(0xab28ecb46814fe75), UINT64_C(0xd1f09c7c5821770c),
-    UINT64_C(0x5e980d24087fec87), UINT64_C(0x24407dec384a65fe),
-    UINT64_C(0x6b1009c7f05548fa), UINT64_C(0x11c8790fc060c183),
-    UINT64_C(0x9ea0e857903e5a08), UINT64_C(0xe478989fa00bd371),
-    UINT64_C(0x7d08ff3b88be6f81), UINT64_C(0x07d08ff3b88be6f8),
-    UINT64_C(0x88b81eabe8d57d73), UINT64_C(0xf2606e63d8e0f40a),
-    UINT64_C(0xbd301a4810ffd90e), UINT64_C(0xc7e86a8020ca5077),
-    UINT64_C(0x4880fbd87094cbfc), UINT64_C(0x32588b1040a14285),
-    UINT64_C(0xd620138fe0aa91f4), UINT64_C(0xacf86347d09f188d),
-    UINT64_C(0x2390f21f80c18306), UINT64_C(0x594882d7b0f40a7f),
-    UINT64_C(0x1618f6fc78eb277b), UINT64_C(0x6cc0863448deae02),
-    UINT64_C(0xe3a8176c18803589), UINT64_C(0x997067a428b5bcf0),
-    UINT64_C(0xfa11fe77117cdf02), UINT64_C(0x80c98ebf2149567b),
-    UINT64_C(0x0fa11fe77117cdf0), UINT64_C(0x75796f2f41224489),
-    UINT64_C(0x3a291b04893d698d), UINT64_C(0x40f16bccb908e0f4),
-    UINT64_C(0xcf99fa94e9567b7f), UINT64_C(0xb5418a5cd963f206),
-    UINT64_C(0x513912c379682177), UINT64_C(0x2be1620b495da80e),
-    UINT64_C(0xa489f35319033385), UINT64_C(0xde51839b2936bafc),
-    UINT64_C(0x9101f7b0e12997f8), UINT64_C(0xebd98778d11c1e81),
-    UINT64_C(0x64b116208142850a), UINT64_C(0x1e6966e8b1770c73),
-    UINT64_C(0x8719014c99c2b083), UINT64_C(0xfdc17184a9f739fa),
-    UINT64_C(0x72a9e0dcf9a9a271), UINT64_C(0x08719014c99c2b08),
-    UINT64_C(0x4721e43f0183060c), UINT64_C(0x3df994f731b68f75),
-    UINT64_C(0xb29105af61e814fe), UINT64_C(0xc849756751dd9d87),
-    UINT64_C(0x2c31edf8f1d64ef6), UINT64_C(0x56e99d30c1e3c78f),
-    UINT64_C(0xd9810c6891bd5c04), UINT64_C(0xa3597ca0a188d57d),
-    UINT64_C(0xec09088b6997f879), UINT64_C(0x96d1784359a27100),
-    UINT64_C(0x19b9e91b09fcea8b), UINT64_C(0x636199d339c963f2),
-    UINT64_C(0xdf7adabd7a6e2d6f), UINT64_C(0xa5a2aa754a5ba416),
-    UINT64_C(0x2aca3b2d1a053f9d), UINT64_C(0x50124be52a30b6e4),
-    UINT64_C(0x1f423fcee22f9be0), UINT64_C(0x659a4f06d21a1299),
-    UINT64_C(0xeaf2de5e82448912), UINT64_C(0x902aae96b271006b),
-    UINT64_C(0x74523609127ad31a), UINT64_C(0x0e8a46c1224f5a63),
-    UINT64_C(0x81e2d7997211c1e8), UINT64_C(0xfb3aa75142244891),
-    UINT64_C(0xb46ad37a8a3b6595), UINT64_C(0xceb2a3b2ba0eecec),
-    UINT64_C(0x41da32eaea507767), UINT64_C(0x3b024222da65fe1e),
-    UINT64_C(0xa2722586f2d042ee), UINT64_C(0xd8aa554ec2e5cb97),
-    UINT64_C(0x57c2c41692bb501c), UINT64_C(0x2d1ab4dea28ed965),
-    UINT64_C(0x624ac0f56a91f461), UINT64_C(0x1892b03d5aa47d18),
-    UINT64_C(0x97fa21650afae693), UINT64_C(0xed2251ad3acf6fea),
-    UINT64_C(0x095ac9329ac4bc9b), UINT64_C(0x7382b9faaaf135e2),
-    UINT64_C(0xfcea28a2faafae69), UINT64_C(0x8632586aca9a2710),
-    UINT64_C(0xc9622c4102850a14), UINT64_C(0xb3ba5c8932b0836d),
-    UINT64_C(0x3cd2cdd162ee18e6), UINT64_C(0x460abd1952db919f),
-    UINT64_C(0x256b24ca6b12f26d), UINT64_C(0x5fb354025b277b14),
-    UINT64_C(0xd0dbc55a0b79e09f), UINT64_C(0xaa03b5923b4c69e6),
-    UINT64_C(0xe553c1b9f35344e2), UINT64_C(0x9f8bb171c366cd9b),
-    UINT64_C(0x10e3202993385610), UINT64_C(0x6a3b50e1a30ddf69),
-    UINT64_C(0x8e43c87e03060c18), UINT64_C(0xf49bb8b633338561),
-    UINT64_C(0x7bf329ee636d1eea), UINT64_C(0x012b592653589793),
-    UINT64_C(0x4e7b2d0d9b47ba97), UINT64_C(0x34a35dc5ab7233ee),
-    UINT64_C(0xbbcbcc9dfb2ca865), UINT64_C(0xc113bc55cb19211c),
-    UINT64_C(0x5863dbf1e3ac9dec), UINT64_C(0x22bbab39d3991495),
-    UINT64_C(0xadd33a6183c78f1e), UINT64_C(0xd70b4aa9b3f20667),
-    UINT64_C(0x985b3e827bed2b63), UINT64_C(0xe2834e4a4bd8a21a),
-    UINT64_C(0x6debdf121b863991), UINT64_C(0x1733afda2bb3b0e8),
-    UINT64_C(0xf34b37458bb86399), UINT64_C(0x8993478dbb8deae0),
-    UINT64_C(0x06fbd6d5ebd3716b), UINT64_C(0x7c23a61ddbe6f812),
-    UINT64_C(0x3373d23613f9d516), UINT64_C(0x49aba2fe23cc5c6f),
-    UINT64_C(0xc6c333a67392c7e4), UINT64_C(0xbc1b436e43a74e9d),
-    UINT64_C(0x95ac9329ac4bc9b5), UINT64_C(0xef74e3e19c7e40cc),
-    UINT64_C(0x601c72b9cc20db47), UINT64_C(0x1ac40271fc15523e),
-    UINT64_C(0x5594765a340a7f3a), UINT64_C(0x2f4c0692043ff643),
-    UINT64_C(0xa02497ca54616dc8), UINT64_C(0xdafce7026454e4b1),
-    UINT64_C(0x3e847f9dc45f37c0), UINT64_C(0x445c0f55f46abeb9),
-    UINT64_C(0xcb349e0da4342532), UINT64_C(0xb1eceec59401ac4b),
-    UINT64_C(0xfebc9aee5c1e814f), UINT64_C(0x8464ea266c2b0836),
-    UINT64_C(0x0b0c7b7e3c7593bd), UINT64_C(0x71d40bb60c401ac4),
-    UINT64_C(0xe8a46c1224f5a634), UINT64_C(0x927c1cda14c02f4d),
-    UINT64_C(0x1d148d82449eb4c6), UINT64_C(0x67ccfd4a74ab3dbf),
-    UINT64_C(0x289c8961bcb410bb), UINT64_C(0x5244f9a98c8199c2),
-    UINT64_C(0xdd2c68f1dcdf0249), UINT64_C(0xa7f41839ecea8b30),
-    UINT64_C(0x438c80a64ce15841), UINT64_C(0x3954f06e7cd4d138),
-    UINT64_C(0xb63c61362c8a4ab3), UINT64_C(0xcce411fe1cbfc3ca),
-    UINT64_C(0x83b465d5d4a0eece), UINT64_C(0xf96c151de49567b7),
-    UINT64_C(0x76048445b4cbfc3c), UINT64_C(0x0cdcf48d84fe7545),
-    UINT64_C(0x6fbd6d5ebd3716b7), UINT64_C(0x15651d968d029fce),
-    UINT64_C(0x9a0d8ccedd5c0445), UINT64_C(0xe0d5fc06ed698d3c),
-    UINT64_C(0xaf85882d2576a038), UINT64_C(0xd55df8e515432941),
-    UINT64_C(0x5a3569bd451db2ca), UINT64_C(0x20ed197575283bb3),
-    UINT64_C(0xc49581ead523e8c2), UINT64_C(0xbe4df122e51661bb),
-    UINT64_C(0x3125607ab548fa30), UINT64_C(0x4bfd10b2857d7349),
-    UINT64_C(0x04ad64994d625e4d), UINT64_C(0x7e7514517d57d734),
-    UINT64_C(0xf11d85092d094cbf), UINT64_C(0x8bc5f5c11d3cc5c6),
-    UINT64_C(0x12b5926535897936), UINT64_C(0x686de2ad05bcf04f),
-    UINT64_C(0xe70573f555e26bc4), UINT64_C(0x9ddd033d65d7e2bd),
-    UINT64_C(0xd28d7716adc8cfb9), UINT64_C(0xa85507de9dfd46c0),
-    UINT64_C(0x273d9686cda3dd4b), UINT64_C(0x5de5e64efd965432),
-    UINT64_C(0xb99d7ed15d9d8743), UINT64_C(0xc3450e196da80e3a),
-    UINT64_C(0x4c2d9f413df695b1), UINT64_C(0x36f5ef890dc31cc8),
-    UINT64_C(0x79a59ba2c5dc31cc), UINT64_C(0x037deb6af5e9b8b5),
-    UINT64_C(0x8c157a32a5b7233e), UINT64_C(0xf6cd0afa9582aa47),
-    UINT64_C(0x4ad64994d625e4da), UINT64_C(0x300e395ce6106da3),
-    UINT64_C(0xbf66a804b64ef628), UINT64_C(0xc5bed8cc867b7f51),
-    UINT64_C(0x8aeeace74e645255), UINT64_C(0xf036dc2f7e51db2c),
-    UINT64_C(0x7f5e4d772e0f40a7), UINT64_C(0x05863dbf1e3ac9de),
-    UINT64_C(0xe1fea520be311aaf), UINT64_C(0x9b26d5e88e0493d6),
-    UINT64_C(0x144e44b0de5a085d), UINT64_C(0x6e963478ee6f8124),
-    UINT64_C(0x21c640532670ac20), UINT64_C(0x5b1e309b16452559),
-    UINT64_C(0xd476a1c3461bbed2), UINT64_C(0xaeaed10b762e37ab),
-    UINT64_C(0x37deb6af5e9b8b5b), UINT64_C(0x4d06c6676eae0222),
-    UINT64_C(0xc26e573f3ef099a9), UINT64_C(0xb8b627f70ec510d0),
-    UINT64_C(0xf7e653dcc6da3dd4), UINT64_C(0x8d3e2314f6efb4ad),
-    UINT64_C(0x0256b24ca6b12f26), UINT64_C(0x788ec2849684a65f),
-    UINT64_C(0x9cf65a1b368f752e), UINT64_C(0xe62e2ad306bafc57),
-    UINT64_C(0x6946bb8b56e467dc), UINT64_C(0x139ecb4366d1eea5),
-    UINT64_C(0x5ccebf68aecec3a1), UINT64_C(0x2616cfa09efb4ad8),
-    UINT64_C(0xa97e5ef8cea5d153), UINT64_C(0xd3a62e30fe90582a),
-    UINT64_C(0xb0c7b7e3c7593bd8), UINT64_C(0xca1fc72bf76cb2a1),
-    UINT64_C(0x45775673a732292a), UINT64_C(0x3faf26bb9707a053),
-    UINT64_C(0x70ff52905f188d57), UINT64_C(0x0a2722586f2d042e),
-    UINT64_C(0x854fb3003f739fa5), UINT64_C(0xff97c3c80f4616dc),
-    UINT64_C(0x1bef5b57af4dc5ad), UINT64_C(0x61372b9f9f784cd4),
-    UINT64_C(0xee5fbac7cf26d75f), UINT64_C(0x9487ca0fff135e26),
-    UINT64_C(0xdbd7be24370c7322), UINT64_C(0xa10fceec0739fa5b),
-    UINT64_C(0x2e675fb4576761d0), UINT64_C(0x54bf2f7c6752e8a9),
-    UINT64_C(0xcdcf48d84fe75459), UINT64_C(0xb71738107fd2dd20),
-    UINT64_C(0x387fa9482f8c46ab), UINT64_C(0x42a7d9801fb9cfd2),
-    UINT64_C(0x0df7adabd7a6e2d6), UINT64_C(0x772fdd63e7936baf),
-    UINT64_C(0xf8474c3bb7cdf024), UINT64_C(0x829f3cf387f8795d),
-    UINT64_C(0x66e7a46c27f3aa2c), UINT64_C(0x1c3fd4a417c62355),
-    UINT64_C(0x935745fc4798b8de), UINT64_C(0xe98f353477ad31a7),
-    UINT64_C(0xa6df411fbfb21ca3), UINT64_C(0xdc0731d78f8795da),
-    UINT64_C(0x536fa08fdfd90e51), UINT64_C(0x29b7d047efec8728),
-};
+/******************** BEGIN GENERATED PYCRC FUNCTIONS ********************/
+/**
+ * Generated on Sun Dec 21 14:14:07 2014,
+ * by pycrc v0.8.2, https://www.tty1.net/pycrc/
+ *
+ * LICENSE ON GENERATED CODE:
+ * ==========================
+ * As of version 0.6, pycrc is released under the terms of the MIT licence.
+ * The code generated by pycrc is not considered a substantial portion of the
+ * software, therefore the author of pycrc will not claim any copyright on
+ * the generated code.
+ * ==========================
+ *
+ * CRC configuration:
+ *    Width        = 64
+ *    Poly         = 0xad93d23594c935a9
+ *    XorIn        = 0xffffffffffffffff
+ *    ReflectIn    = True
+ *    XorOut       = 0x0000000000000000
+ *    ReflectOut   = True
+ *    Algorithm    = bit-by-bit-fast
+ *
+ * Modifications after generation (by matt):
+ *   - included finalize step in-line with update for single-call generation
+ *   - re-worked some inner variable architectures
+ *   - adjusted function parameters to match expected prototypes.
+ *****************************************************************************/
 
-uint64_t crc64(uint64_t crc, const unsigned char *s, uint64_t l) {
-    uint64_t j;
+/**
+ * Reflect all bits of a \a data word of \a data_len bytes.
+ *
+ * \param data         The data word to be reflected.
+ * \param data_len     The width of \a data expressed in number of bits.
+ * \return             The reflected data.
+ *****************************************************************************/
+static inline uint_fast64_t crc_reflect(uint_fast64_t data, size_t data_len) {
+    uint_fast64_t ret = data & 0x01;
 
-    for (j = 0; j < l; j++) {
-        uint8_t byte = s[j];
-        crc = crc64_tab[(uint8_t)crc ^ byte] ^ (crc >> 8);
+    for (size_t i = 1; i < data_len; i++) {
+        data >>= 1;
+        ret = (ret << 1) | (data & 0x01);
     }
-    return crc;
+
+    return ret;
+}
+
+/**
+ *  Update the crc value with new data.
+ *
+ * \param crc      The current crc value.
+ * \param data     Pointer to a buffer of \a data_len bytes.
+ * \param data_len Number of bytes in the \a data buffer.
+ * \return         The updated crc value.
+ ******************************************************************************/
+uint64_t _crc64(uint_fast64_t crc, const void *in_data, const uint64_t len) {
+    const uint8_t *data = in_data;
+    unsigned long long bit;
+
+    for (uint64_t offset = 0; offset < len; offset++) {
+        uint8_t c = data[offset];
+        for (uint_fast8_t i = 0x01; i & 0xff; i <<= 1) {
+            bit = crc & 0x8000000000000000;
+            if (c & i) {
+                bit = !bit;
+            }
+
+            crc <<= 1;
+            if (bit) {
+                crc ^= POLY;
+            }
+        }
+
+        crc &= 0xffffffffffffffff;
+    }
+
+    crc = crc & 0xffffffffffffffff;
+    return crc_reflect(crc, 64) ^ 0x0000000000000000;
+}
+
+/******************** END GENERATED PYCRC FUNCTIONS ********************/
+
+/* Initializes the 16KB lookup tables. */
+void crc64_init(void) {
+    crcspeed64native_init(_crc64, crc64_table);
+}
+
+/* Compute crc64 */
+uint64_t crc64(uint64_t crc, const unsigned char *s, uint64_t l) {
+    return crcspeed64native(crc64_table, crc, (void *) s, l);
 }
 
 /* Test main */
-#ifdef REDIS_TEST
+#if defined(REDIS_TEST) || defined(REDIS_TEST_MAIN)
 #include <stdio.h>
 
 #define UNUSED(x) (void)(x)
 int crc64Test(int argc, char *argv[]) {
     UNUSED(argc);
     UNUSED(argv);
-    printf("e9c6d914c4b8d9ca == %016llx\n",
-        (unsigned long long) crc64(0,(unsigned char*)"123456789",9));
+    crc64_init();
+    printf("[calcula]: e9c6d914c4b8d9ca == %016" PRIx64 "\n",
+           (uint64_t)_crc64(0, "123456789", 9));
+    printf("[64speed]: e9c6d914c4b8d9ca == %016" PRIx64 "\n",
+           (uint64_t)crc64speed(0, "123456789", 9));
+    char li[] = "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed "
+                "do eiusmod tempor incididunt ut labore et dolore magna "
+                "aliqua. Ut enim ad minim veniam, quis nostrud exercitation "
+                "ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis "
+                "aute irure dolor in reprehenderit in voluptate velit esse "
+                "cillum dolore eu fugiat nulla pariatur. Excepteur sint "
+                "occaecat cupidatat non proident, sunt in culpa qui officia "
+                "deserunt mollit anim id est laborum.";
+    printf("[calcula]: c7794709e69683b3 == %016" PRIx64 "\n",
+           (uint64_t)_crc64(0, li, sizeof(li)));
+    printf("[64speed]: c7794709e69683b3 == %016" PRIx64 "\n",
+           (uint64_t)crc64(0, li, sizeof(li)));
     return 0;
 }
+
+#endif
+
+#ifdef REDIS_TEST_MAIN
+int main(int argc, char *argv[]) {
+    return crc64Test(argc, argv);
+}
+
 #endif
diff --git a/src/crc64.h b/src/crc64.h
index c9fca519d..60c42345f 100644
--- a/src/crc64.h
+++ b/src/crc64.h
@@ -3,6 +3,7 @@
 
 #include <stdint.h>
 
+void crc64_init(void);
 uint64_t crc64(uint64_t crc, const unsigned char *s, uint64_t l);
 
 #ifdef REDIS_TEST
diff --git a/src/rdb.c b/src/rdb.c
index 143b6c325..9f6bf13f1 100644
--- a/src/rdb.c
+++ b/src/rdb.c
@@ -2291,7 +2291,8 @@ int rdbLoadRio(rio *rdb, int rdbflags, rdbSaveInfo *rsi) {
             if (cksum == 0) {
                 serverLog(LL_WARNING,"RDB file was saved with checksum disabled: no check performed.");
             } else if (cksum != expected) {
-                serverLog(LL_WARNING,"Wrong RDB checksum. Aborting now.");
+                serverLog(LL_WARNING,"Wrong RDB checksum expected: (%llx) but "
+                    "got (%llx). Aborting now.",expected,cksum);
                 rdbExitReportCorruptRDB("RDB CRC error");
             }
         }
diff --git a/src/server.c b/src/server.c
index db1d3780c..e265fe9ab 100644
--- a/src/server.c
+++ b/src/server.c
@@ -2892,6 +2892,7 @@ void initServer(void) {
     scriptingInit(1);
     slowlogInit();
     latencyMonitorInit();
+    crc64_init();
 }
 
 /* Some steps in server initialization need to be done last (after modules

From a75fa3aad1b6cdefd72e1586ea5a51bd33569ba1 Mon Sep 17 00:00:00 2001
From: Madelyn Olson <matolson@amazon.com>
Date: Fri, 24 Apr 2020 17:05:37 -0700
Subject: [PATCH 096/186] Made crc64 test consistent

---
 src/crc64.c | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/crc64.c b/src/crc64.c
index 165941dea..4cbc019f6 100644
--- a/src/crc64.c
+++ b/src/crc64.c
@@ -30,6 +30,7 @@
 #include "crcspeed.h"
 static uint64_t crc64_table[8][256] = {{0}};
 
+#define POLY UINT64_C(0xad93d23594c935a9)
 /******************** BEGIN GENERATED PYCRC FUNCTIONS ********************/
 /**
  * Generated on Sun Dec 21 14:14:07 2014,
@@ -122,7 +123,7 @@ uint64_t crc64(uint64_t crc, const unsigned char *s, uint64_t l) {
 }
 
 /* Test main */
-#if defined(REDIS_TEST) || defined(REDIS_TEST_MAIN)
+#ifdef REDIS_TEST
 #include <stdio.h>
 
 #define UNUSED(x) (void)(x)
@@ -133,7 +134,7 @@ int crc64Test(int argc, char *argv[]) {
     printf("[calcula]: e9c6d914c4b8d9ca == %016" PRIx64 "\n",
            (uint64_t)_crc64(0, "123456789", 9));
     printf("[64speed]: e9c6d914c4b8d9ca == %016" PRIx64 "\n",
-           (uint64_t)crc64speed(0, "123456789", 9));
+           (uint64_t)crc64(0, "123456789", 9));
     char li[] = "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed "
                 "do eiusmod tempor incididunt ut labore et dolore magna "
                 "aliqua. Ut enim ad minim veniam, quis nostrud exercitation "

From a1bed447b3ad2faae91f51f9c8e3f2813ae3f2e4 Mon Sep 17 00:00:00 2001
From: Madelyn Olson <matolson@amazon.com>
Date: Fri, 24 Apr 2020 17:11:21 -0700
Subject: [PATCH 097/186] Added crcspeed library

---
 src/crcspeed.c | 281 +++++++++++++++++++++++++++++++++++++++++++++++++
 src/crcspeed.h |  60 +++++++++++
 2 files changed, 341 insertions(+)
 create mode 100644 src/crcspeed.c
 create mode 100644 src/crcspeed.h

diff --git a/src/crcspeed.c b/src/crcspeed.c
new file mode 100644
index 000000000..d2d97a8c7
--- /dev/null
+++ b/src/crcspeed.c
@@ -0,0 +1,281 @@
+/*
+ * Copyright (C) 2013 Mark Adler
+ * Originally by: crc64.c Version 1.4  16 Dec 2013  Mark Adler
+ * Modifications by Matt Stancliff <matt@genges.com>:
+ *   - removed CRC64-specific behavior
+ *   - added generation of lookup tables by parameters
+ *   - removed inversion of CRC input/result
+ *   - removed automatic initialization in favor of explicit initialization
+
+  This software is provided 'as-is', without any express or implied
+  warranty.  In no event will the author be held liable for any damages
+  arising from the use of this software.
+
+  Permission is granted to anyone to use this software for any purpose,
+  including commercial applications, and to alter it and redistribute it
+  freely, subject to the following restrictions:
+
+  1. The origin of this software must not be misrepresented; you must not
+     claim that you wrote the original software. If you use this software
+     in a product, an acknowledgment in the product documentation would be
+     appreciated but is not required.
+  2. Altered source versions must be plainly marked as such, and must not be
+     misrepresented as being the original software.
+  3. This notice may not be removed or altered from any source distribution.
+
+  Mark Adler
+  madler@alumni.caltech.edu
+ */
+
+#include "crcspeed.h"
+
+/* Fill in a CRC constants table. */
+void crcspeed64little_init(crcfn64 crcfn, uint64_t table[8][256]) {
+    uint64_t crc;
+
+    /* generate CRCs for all single byte sequences */
+    for (int n = 0; n < 256; n++) {
+        table[0][n] = crcfn(0, &n, 1);
+    }
+
+    /* generate nested CRC table for future slice-by-8 lookup */
+    for (int n = 0; n < 256; n++) {
+        crc = table[0][n];
+        for (int k = 1; k < 8; k++) {
+            crc = table[0][crc & 0xff] ^ (crc >> 8);
+            table[k][n] = crc;
+        }
+    }
+}
+
+void crcspeed16little_init(crcfn16 crcfn, uint16_t table[8][256]) {
+    uint16_t crc;
+
+    /* generate CRCs for all single byte sequences */
+    for (int n = 0; n < 256; n++) {
+        table[0][n] = crcfn(0, &n, 1);
+    }
+
+    /* generate nested CRC table for future slice-by-8 lookup */
+    for (int n = 0; n < 256; n++) {
+        crc = table[0][n];
+        for (int k = 1; k < 8; k++) {
+            crc = table[0][(crc >> 8) & 0xff] ^ (crc << 8);
+            table[k][n] = crc;
+        }
+    }
+}
+
+/* Reverse the bytes in a 64-bit word. */
+static inline uint64_t rev8(uint64_t a) {
+#if defined(__GNUC__) || defined(__clang__)
+    return __builtin_bswap64(a);
+#else
+    uint64_t m;
+
+    m = UINT64_C(0xff00ff00ff00ff);
+    a = ((a >> 8) & m) | (a & m) << 8;
+    m = UINT64_C(0xffff0000ffff);
+    a = ((a >> 16) & m) | (a & m) << 16;
+    return a >> 32 | a << 32;
+#endif
+}
+
+/* This function is called once to initialize the CRC table for use on a
+   big-endian architecture. */
+void crcspeed64big_init(crcfn64 fn, uint64_t big_table[8][256]) {
+    /* Create the little endian table then reverse all the entires. */
+    crcspeed64little_init(fn, big_table);
+    for (int k = 0; k < 8; k++) {
+        for (int n = 0; n < 256; n++) {
+            big_table[k][n] = rev8(big_table[k][n]);
+        }
+    }
+}
+
+void crcspeed16big_init(crcfn16 fn, uint16_t big_table[8][256]) {
+    /* Create the little endian table then reverse all the entires. */
+    crcspeed16little_init(fn, big_table);
+    for (int k = 0; k < 8; k++) {
+        for (int n = 0; n < 256; n++) {
+            big_table[k][n] = rev8(big_table[k][n]);
+        }
+    }
+}
+
+/* Calculate a non-inverted CRC multiple bytes at a time on a little-endian
+ * architecture. If you need inverted CRC, invert *before* calling and invert
+ * *after* calling.
+ * 64 bit crc = process 8 bytes at once;
+ */
+uint64_t crcspeed64little(uint64_t little_table[8][256], uint64_t crc,
+                          void *buf, size_t len) {
+    unsigned char *next = buf;
+
+    /* process individual bytes until we reach an 8-byte aligned pointer */
+    while (len && ((uintptr_t)next & 7) != 0) {
+        crc = little_table[0][(crc ^ *next++) & 0xff] ^ (crc >> 8);
+        len--;
+    }
+
+    /* fast middle processing, 8 bytes (aligned!) per loop */
+    while (len >= 8) {
+        crc ^= *(uint64_t *)next;
+        crc = little_table[7][crc & 0xff] ^
+              little_table[6][(crc >> 8) & 0xff] ^
+              little_table[5][(crc >> 16) & 0xff] ^
+              little_table[4][(crc >> 24) & 0xff] ^
+              little_table[3][(crc >> 32) & 0xff] ^
+              little_table[2][(crc >> 40) & 0xff] ^
+              little_table[1][(crc >> 48) & 0xff] ^
+              little_table[0][crc >> 56];
+        next += 8;
+        len -= 8;
+    }
+
+    /* process remaining bytes (can't be larger than 8) */
+    while (len) {
+        crc = little_table[0][(crc ^ *next++) & 0xff] ^ (crc >> 8);
+        len--;
+    }
+
+    return crc;
+}
+
+uint16_t crcspeed16little(uint16_t little_table[8][256], uint16_t crc,
+                          void *buf, size_t len) {
+    unsigned char *next = buf;
+
+    /* process individual bytes until we reach an 8-byte aligned pointer */
+    while (len && ((uintptr_t)next & 7) != 0) {
+        crc = little_table[0][((crc >> 8) ^ *next++) & 0xff] ^ (crc << 8);
+        len--;
+    }
+
+    /* fast middle processing, 8 bytes (aligned!) per loop */
+    while (len >= 8) {
+        uint64_t n = *(uint64_t *)next;
+        crc = little_table[7][(n & 0xff) ^ ((crc >> 8) & 0xff)] ^
+              little_table[6][((n >> 8) & 0xff) ^ (crc & 0xff)] ^
+              little_table[5][(n >> 16) & 0xff] ^
+              little_table[4][(n >> 24) & 0xff] ^
+              little_table[3][(n >> 32) & 0xff] ^
+              little_table[2][(n >> 40) & 0xff] ^
+              little_table[1][(n >> 48) & 0xff] ^
+              little_table[0][n >> 56];
+        next += 8;
+        len -= 8;
+    }
+
+    /* process remaining bytes (can't be larger than 8) */
+    while (len) {
+        crc = little_table[0][((crc >> 8) ^ *next++) & 0xff] ^ (crc << 8);
+        len--;
+    }
+
+    return crc;
+}
+
+/* Calculate a non-inverted CRC eight bytes at a time on a big-endian
+ * architecture.
+ */
+uint64_t crcspeed64big(uint64_t big_table[8][256], uint64_t crc, void *buf,
+                       size_t len) {
+    unsigned char *next = buf;
+
+    crc = rev8(crc);
+    while (len && ((uintptr_t)next & 7) != 0) {
+        crc = big_table[0][(crc >> 56) ^ *next++] ^ (crc << 8);
+        len--;
+    }
+
+    while (len >= 8) {
+        crc ^= *(uint64_t *)next;
+        crc = big_table[0][crc & 0xff] ^
+              big_table[1][(crc >> 8) & 0xff] ^
+              big_table[2][(crc >> 16) & 0xff] ^
+              big_table[3][(crc >> 24) & 0xff] ^
+              big_table[4][(crc >> 32) & 0xff] ^
+              big_table[5][(crc >> 40) & 0xff] ^
+              big_table[6][(crc >> 48) & 0xff] ^
+              big_table[7][crc >> 56];
+        next += 8;
+        len -= 8;
+    }
+
+    while (len) {
+        crc = big_table[0][(crc >> 56) ^ *next++] ^ (crc << 8);
+        len--;
+    }
+
+    return rev8(crc);
+}
+
+/* WARNING: Completely untested on big endian architecture.  Possibly broken. */
+uint16_t crcspeed16big(uint16_t big_table[8][256], uint16_t crc_in, void *buf,
+                       size_t len) {
+    unsigned char *next = buf;
+    uint64_t crc = crc_in;
+
+    crc = rev8(crc);
+    while (len && ((uintptr_t)next & 7) != 0) {
+        crc = big_table[0][((crc >> (56 - 8)) ^ *next++) & 0xff] ^ (crc >> 8);
+        len--;
+    }
+
+    while (len >= 8) {
+        uint64_t n = *(uint64_t *)next;
+        crc = big_table[0][(n & 0xff) ^ ((crc >> (56 - 8)) & 0xff)] ^
+              big_table[1][((n >> 8) & 0xff) ^ (crc & 0xff)] ^
+              big_table[2][(n >> 16) & 0xff] ^
+              big_table[3][(n >> 24) & 0xff] ^
+              big_table[4][(n >> 32) & 0xff] ^
+              big_table[5][(n >> 40) & 0xff] ^
+              big_table[6][(n >> 48) & 0xff] ^
+              big_table[7][n >> 56];
+        next += 8;
+        len -= 8;
+    }
+
+    while (len) {
+        crc = big_table[0][((crc >> (56 - 8)) ^ *next++) & 0xff] ^ (crc >> 8);
+        len--;
+    }
+
+    return rev8(crc);
+}
+
+/* Return the CRC of buf[0..len-1] with initial crc, processing eight bytes
+   at a time using passed-in lookup table.
+   This selects one of two routines depending on the endianess of
+   the architecture. */
+uint64_t crcspeed64native(uint64_t table[8][256], uint64_t crc, void *buf,
+                          size_t len) {
+    uint64_t n = 1;
+
+    return *(char *)&n ? crcspeed64little(table, crc, buf, len)
+                       : crcspeed64big(table, crc, buf, len);
+}
+
+uint16_t crcspeed16native(uint16_t table[8][256], uint16_t crc, void *buf,
+                          size_t len) {
+    uint64_t n = 1;
+
+    return *(char *)&n ? crcspeed16little(table, crc, buf, len)
+                       : crcspeed16big(table, crc, buf, len);
+}
+
+/* Initialize CRC lookup table in architecture-dependent manner. */
+void crcspeed64native_init(crcfn64 fn, uint64_t table[8][256]) {
+    uint64_t n = 1;
+
+    *(char *)&n ? crcspeed64little_init(fn, table)
+                : crcspeed64big_init(fn, table);
+}
+
+void crcspeed16native_init(crcfn16 fn, uint16_t table[8][256]) {
+    uint64_t n = 1;
+
+    *(char *)&n ? crcspeed16little_init(fn, table)
+                : crcspeed16big_init(fn, table);
+}
diff --git a/src/crcspeed.h b/src/crcspeed.h
new file mode 100644
index 000000000..d7ee95ebb
--- /dev/null
+++ b/src/crcspeed.h
@@ -0,0 +1,60 @@
+/* Copyright (c) 2014, Matt Stancliff <matt@genges.com>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *   * Redistributions of source code must retain the above copyright notice,
+ *     this list of conditions and the following disclaimer.
+ *   * Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *   * Neither the name of Redis nor the names of its contributors may be used
+ *     to endorse or promote products derived from this software without
+ *     specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE. */
+
+#ifndef CRCSPEED_H
+#define CRCSPEED_H
+
+#include <inttypes.h>
+#include <stdio.h>
+
+typedef uint64_t (*crcfn64)(uint64_t, const void *, const uint64_t);
+typedef uint16_t (*crcfn16)(uint16_t, const void *, const uint64_t);
+
+/* CRC-64 */
+void crcspeed64little_init(crcfn64 fn, uint64_t table[8][256]);
+void crcspeed64big_init(crcfn64 fn, uint64_t table[8][256]);
+void crcspeed64native_init(crcfn64 fn, uint64_t table[8][256]);
+
+uint64_t crcspeed64little(uint64_t table[8][256], uint64_t crc, void *buf,
+                          size_t len);
+uint64_t crcspeed64big(uint64_t table[8][256], uint64_t crc, void *buf,
+                       size_t len);
+uint64_t crcspeed64native(uint64_t table[8][256], uint64_t crc, void *buf,
+                          size_t len);
+
+/* CRC-16 */
+void crcspeed16little_init(crcfn16 fn, uint16_t table[8][256]);
+void crcspeed16big_init(crcfn16 fn, uint16_t table[8][256]);
+void crcspeed16native_init(crcfn16 fn, uint16_t table[8][256]);
+
+uint16_t crcspeed16little(uint16_t table[8][256], uint16_t crc, void *buf,
+                          size_t len);
+uint16_t crcspeed16big(uint16_t table[8][256], uint16_t crc, void *buf,
+                       size_t len);
+uint16_t crcspeed16native(uint16_t table[8][256], uint16_t crc, void *buf,
+                          size_t len);
+#endif

From 0d1e8c93bf9cf8431dd4ca90817150a786c7a5d0 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Mon, 27 Apr 2020 23:17:19 +0300
Subject: [PATCH 098/186] allow dictFind using static robj

since the recent addition of OBJ_STATIC_REFCOUNT and the assertion in
incrRefCount it is now impossible to use dictFind using a static robj,
because dictEncObjKeyCompare will call getDecodedObject which tries to
increment the refcount just in order to decrement it later.
---
 src/server.c | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/src/server.c b/src/server.c
index e265fe9ab..1bfca1a0b 100644
--- a/src/server.c
+++ b/src/server.c
@@ -1219,11 +1219,16 @@ int dictEncObjKeyCompare(void *privdata, const void *key1,
         o2->encoding == OBJ_ENCODING_INT)
             return o1->ptr == o2->ptr;
 
-    o1 = getDecodedObject(o1);
-    o2 = getDecodedObject(o2);
+    /* due to OBJ_STATIC_REFCOUNT, we rather not call sdsEncodedObject unnecessarily */
+    if (!sdsEncodedObject(o1))
+        o1 = getDecodedObject(o1);
+    if (!sdsEncodedObject(o2))
+        o2 = getDecodedObject(o2);
     cmp = dictSdsKeyCompare(privdata,o1->ptr,o2->ptr);
-    decrRefCount(o1);
-    decrRefCount(o2);
+    if (o1!=key1)
+        decrRefCount(o1);
+    if (o2!=key2)
+        decrRefCount(o2);
     return cmp;
 }
 

From 64e588bfabcd0cf0a8f862276fd7ea07e00756c3 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Mon, 27 Apr 2020 22:40:15 +0200
Subject: [PATCH 099/186] Rework comment in dictEncObjKeyCompare().

---
 src/server.c | 17 ++++++++---------
 1 file changed, 8 insertions(+), 9 deletions(-)

diff --git a/src/server.c b/src/server.c
index 1bfca1a0b..1086dab48 100644
--- a/src/server.c
+++ b/src/server.c
@@ -1219,16 +1219,15 @@ int dictEncObjKeyCompare(void *privdata, const void *key1,
         o2->encoding == OBJ_ENCODING_INT)
             return o1->ptr == o2->ptr;
 
-    /* due to OBJ_STATIC_REFCOUNT, we rather not call sdsEncodedObject unnecessarily */
-    if (!sdsEncodedObject(o1))
-        o1 = getDecodedObject(o1);
-    if (!sdsEncodedObject(o2))
-        o2 = getDecodedObject(o2);
+    /* Due to OBJ_STATIC_REFCOUNT, we avoid calling getDecodedObject() without
+     * good reasons, because it would incrRefCount() the object, which
+     * is invalid. So we check to make sure dictFind() works with static
+     * objects as well. */
+    if (!sdsEncodedObject(o1)) o1 = getDecodedObject(o1);
+    if (!sdsEncodedObject(o2)) o2 = getDecodedObject(o2);
     cmp = dictSdsKeyCompare(privdata,o1->ptr,o2->ptr);
-    if (o1!=key1)
-        decrRefCount(o1);
-    if (o2!=key2)
-        decrRefCount(o2);
+    if (o1!=key1) decrRefCount(o1);
+    if (o2!=key2) decrRefCount(o2);
     return cmp;
 }
 

From ea63aea72d829a330a904668c847e600debbe626 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Tue, 28 Apr 2020 09:18:01 +0300
Subject: [PATCH 100/186] fix loading race in psync2 tests

---
 tests/integration/psync2-pingoff.tcl |  1 +
 tests/integration/psync2-reg.tcl     |  5 ++++-
 tests/integration/psync2.tcl         | 10 ++++++++++
 3 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/tests/integration/psync2-pingoff.tcl b/tests/integration/psync2-pingoff.tcl
index 1cea290e7..420747d21 100644
--- a/tests/integration/psync2-pingoff.tcl
+++ b/tests/integration/psync2-pingoff.tcl
@@ -20,6 +20,7 @@ start_server {} {
         $R(1) replicaof $R_host(0) $R_port(0)
         $R(0) set foo bar
         wait_for_condition 50 1000 {
+            [status $R(1) master_link_status] == "up" &&
             [$R(0) dbsize] == 1 && [$R(1) dbsize] == 1
         } else {
             fail "Replicas not replicating from master"
diff --git a/tests/integration/psync2-reg.tcl b/tests/integration/psync2-reg.tcl
index b5ad021e2..71a1c0eb2 100644
--- a/tests/integration/psync2-reg.tcl
+++ b/tests/integration/psync2-reg.tcl
@@ -28,7 +28,10 @@ start_server {} {
         $R(2) slaveof $R_host(0) $R_port(0)
         $R(0) set foo bar
         wait_for_condition 50 1000 {
-            [$R(1) dbsize] == 1 && [$R(2) dbsize] == 1
+            [status $R(1) master_link_status] == "up" &&
+            [status $R(2) master_link_status] == "up" &&
+            [$R(1) dbsize] == 1 &&
+            [$R(2) dbsize] == 1
         } else {
             fail "Replicas not replicating from master"
         }
diff --git a/tests/integration/psync2.tcl b/tests/integration/psync2.tcl
index 4e1189e0b..5fe29caba 100644
--- a/tests/integration/psync2.tcl
+++ b/tests/integration/psync2.tcl
@@ -67,6 +67,16 @@ start_server {} {
             lappend used $slave_id
         }
 
+        # Wait for replicas to sync. so next loop won't get -LOADING error
+        wait_for_condition 50 1000 {
+            [status $R([expr {($master_id+1)%5}]) master_link_status] == "up" &&
+            [status $R([expr {($master_id+2)%5}]) master_link_status] == "up" &&
+            [status $R([expr {($master_id+3)%5}]) master_link_status] == "up" &&
+            [status $R([expr {($master_id+4)%5}]) master_link_status] == "up"
+        } else {
+            fail "Replica not reconnecting"
+        }
+
         # 3) Increment the counter and wait for all the instances
         # to converge.
         test "PSYNC2: cluster is consistent after failover" {

From b712fba17ccf4c818a2c795e660f9cbbb9c80bfe Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Tue, 28 Apr 2020 12:14:46 +0300
Subject: [PATCH 101/186] hickup, re-fix dictEncObjKeyCompare

come to think of it, in theory (not in practice), getDecodedObject can
return the same original object with refcount incremented, so the
pointer comparision in the previous commit was invalid.
so now instead of checking the encoding, we explicitly check the
refcount.
---
 src/server.c | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/server.c b/src/server.c
index 1086dab48..ab6280121 100644
--- a/src/server.c
+++ b/src/server.c
@@ -1223,11 +1223,11 @@ int dictEncObjKeyCompare(void *privdata, const void *key1,
      * good reasons, because it would incrRefCount() the object, which
      * is invalid. So we check to make sure dictFind() works with static
      * objects as well. */
-    if (!sdsEncodedObject(o1)) o1 = getDecodedObject(o1);
-    if (!sdsEncodedObject(o2)) o2 = getDecodedObject(o2);
+    if (o1->refcount != OBJ_STATIC_REFCOUNT) o1 = getDecodedObject(o1);
+    if (o2->refcount != OBJ_STATIC_REFCOUNT) o2 = getDecodedObject(o2);
     cmp = dictSdsKeyCompare(privdata,o1->ptr,o2->ptr);
-    if (o1!=key1) decrRefCount(o1);
-    if (o2!=key2) decrRefCount(o2);
+    if (o1->refcount != OBJ_STATIC_REFCOUNT) decrRefCount(o1);
+    if (o2->refcount != OBJ_STATIC_REFCOUNT) decrRefCount(o2);
     return cmp;
 }
 

From cac9d7cf7e93ff2e3a8b3092134a24988e7336da Mon Sep 17 00:00:00 2001
From: Itamar Haber <itamar@redislabs.com>
Date: Fri, 28 Feb 2020 13:35:10 +0200
Subject: [PATCH 102/186] Adds `BIN_PATH` to create-cluster

Allows for setting the binaries path if used outside the upstream repo.

Also documents `call` in usage clause (TODO: port to
`redis-cli --cluster call` or just deprecate it).
---
 utils/create-cluster/create-cluster | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/utils/create-cluster/create-cluster b/utils/create-cluster/create-cluster
index 4aef0816a..5b8bfe250 100755
--- a/utils/create-cluster/create-cluster
+++ b/utils/create-cluster/create-cluster
@@ -1,6 +1,7 @@
 #!/bin/bash
 
 # Settings
+BIN_PATH="../../"
 CLUSTER_HOST=127.0.0.1
 PORT=30000
 TIMEOUT=2000
@@ -25,7 +26,7 @@ then
     while [ $((PORT < ENDPORT)) != "0" ]; do
         PORT=$((PORT+1))
         echo "Starting $PORT"
-        ../../src/redis-server --port $PORT  --protected-mode $PROTECTED_MODE --cluster-enabled yes --cluster-config-file nodes-${PORT}.conf --cluster-node-timeout $TIMEOUT --appendonly yes --appendfilename appendonly-${PORT}.aof --dbfilename dump-${PORT}.rdb --logfile ${PORT}.log --daemonize yes ${ADDITIONAL_OPTIONS}
+        $BIN_PATH/redis-server --port $PORT  --protected-mode $PROTECTED_MODE --cluster-enabled yes --cluster-config-file nodes-${PORT}.conf --cluster-node-timeout $TIMEOUT --appendonly yes --appendfilename appendonly-${PORT}.aof --dbfilename dump-${PORT}.rdb --logfile ${PORT}.log --daemonize yes ${ADDITIONAL_OPTIONS}
     done
     exit 0
 fi
@@ -37,7 +38,7 @@ then
         PORT=$((PORT+1))
         HOSTS="$HOSTS $CLUSTER_HOST:$PORT"
     done
-    ../../src/redis-cli --cluster create $HOSTS --cluster-replicas $REPLICAS
+    $BIN_PATH/redis-cli --cluster create $HOSTS --cluster-replicas $REPLICAS
     exit 0
 fi
 
@@ -46,7 +47,7 @@ then
     while [ $((PORT < ENDPORT)) != "0" ]; do
         PORT=$((PORT+1))
         echo "Stopping $PORT"
-        ../../src/redis-cli -p $PORT shutdown nosave
+        $BIN_PATH/redis-cli -p $PORT shutdown nosave
     done
     exit 0
 fi
@@ -57,7 +58,7 @@ then
     while [ 1 ]; do
         clear
         date
-        ../../src/redis-cli -p $PORT cluster nodes | head -30
+        $BIN_PATH/redis-cli -p $PORT cluster nodes | head -30
         sleep 1
     done
     exit 0
@@ -81,7 +82,7 @@ if [ "$1" == "call" ]
 then
     while [ $((PORT < ENDPORT)) != "0" ]; do
         PORT=$((PORT+1))
-        ../../src/redis-cli -p $PORT $2 $3 $4 $5 $6 $7 $8 $9
+        $BIN_PATH/redis-cli -p $PORT $2 $3 $4 $5 $6 $7 $8 $9
     done
     exit 0
 fi
@@ -101,7 +102,7 @@ then
     exit 0
 fi
 
-echo "Usage: $0 [start|create|stop|watch|tail|clean]"
+echo "Usage: $0 [start|create|stop|watch|tail|clean|call]"
 echo "start       -- Launch Redis Cluster instances."
 echo "create      -- Create a cluster using redis-cli --cluster create."
 echo "stop        -- Stop Redis Cluster instances."
@@ -110,3 +111,4 @@ echo "tail <id>   -- Run tail -f of instance at base port + ID."
 echo "tailall     -- Run tail -f for all the log files at once."
 echo "clean       -- Remove all instances data, logs, configs."
 echo "clean-logs  -- Remove just instances logs."
+echo "call <cmd>  -- Call a command (up to 7 arguments) on all nodes. "

From 56d628f8512e59a149c026ce005e35e9aa3197b9 Mon Sep 17 00:00:00 2001
From: Itamar Haber <itamar@redislabs.com>
Date: Fri, 28 Feb 2020 13:36:50 +0200
Subject: [PATCH 103/186] Update create-cluster

---
 utils/create-cluster/create-cluster | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/utils/create-cluster/create-cluster b/utils/create-cluster/create-cluster
index 5b8bfe250..4db0a6619 100755
--- a/utils/create-cluster/create-cluster
+++ b/utils/create-cluster/create-cluster
@@ -111,4 +111,4 @@ echo "tail <id>   -- Run tail -f of instance at base port + ID."
 echo "tailall     -- Run tail -f for all the log files at once."
 echo "clean       -- Remove all instances data, logs, configs."
 echo "clean-logs  -- Remove just instances logs."
-echo "call <cmd>  -- Call a command (up to 7 arguments) on all nodes. "
+echo "call <cmd>  -- Call a command (up to 7 arguments) on all nodes."

From 5bfc18950a4812fe6961904c1b1cae109f4e9b71 Mon Sep 17 00:00:00 2001
From: hwware <wen.hui.ware@gmail.com>
Date: Wed, 15 Apr 2020 22:00:36 -0400
Subject: [PATCH 104/186] Fix not used marco in cluster.c

---
 src/cluster.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/cluster.c b/src/cluster.c
index e3adaf83a..49a410d54 100644
--- a/src/cluster.c
+++ b/src/cluster.c
@@ -2106,7 +2106,7 @@ int clusterProcessPacket(clusterLink *link) {
         resetManualFailover();
         server.cluster->mf_end = mstime() + CLUSTER_MF_TIMEOUT;
         server.cluster->mf_slave = sender;
-        pauseClients(mstime()+(CLUSTER_MF_TIMEOUT*2));
+        pauseClients(mstime()+(CLUSTER_MF_TIMEOUT*CLUSTER_MF_PAUSE_MULT));
         serverLog(LL_WARNING,"Manual failover requested by replica %.40s.",
             sender->name);
     } else if (type == CLUSTERMSG_TYPE_UPDATE) {

From 6c0bc608a19a5071fca7f755fccd4ed679619195 Mon Sep 17 00:00:00 2001
From: Guy Benoish <guy.benoish@redislabs.com>
Date: Wed, 22 Apr 2020 16:05:57 +0300
Subject: [PATCH 105/186] Extend XINFO STREAM output

Introducing XINFO STREAM <key> FULL
---
 src/t_stream.c                     | 226 ++++++++++++++++++++++++-----
 tests/unit/type/stream-cgroups.tcl |  34 +++++
 2 files changed, 226 insertions(+), 34 deletions(-)

diff --git a/src/t_stream.c b/src/t_stream.c
index 758db637e..d6a5e0009 100644
--- a/src/t_stream.c
+++ b/src/t_stream.c
@@ -2489,16 +2489,200 @@ void xtrimCommand(client *c) {
     addReplyLongLong(c,deleted);
 }
 
+/* Helper function for xinfoCommand.
+ * Handles the variants of XINFO STREAM */
+void xinfoReplyWithStreamInfo(client *c, stream *s) {
+    int full = 1;
+    long long count = 0;
+    robj **optv = c->argv + 3; /* Options start after XINFO STREAM <key> */
+    int optc = c->argc - 3;
+
+    /* Parse options. */
+    if (optc == 0) {
+        full = 0;
+    } else {
+        /* Valid options are [FULL] or [FULL COUNT <count>] */
+        if (optc != 1 && optc != 3) {
+            addReplySubcommandSyntaxError(c);
+            return;
+        }
+
+        /* First option must be "FULL" */
+        if (strcasecmp(optv[0]->ptr,"full")) {
+            addReplySubcommandSyntaxError(c);
+            return;
+        }
+
+        if (optc == 3) {
+            /* First option must be "FULL" */
+            if (strcasecmp(optv[1]->ptr,"count")) {
+                addReplySubcommandSyntaxError(c);
+                return;
+            }
+            if (getLongLongFromObjectOrReply(c,optv[2],&count,NULL) == C_ERR)
+                return;
+            if (count < 0) count = 0;
+        }
+    }
+
+    addReplyMapLen(c,full ? 6 : 7);
+    addReplyBulkCString(c,"length");
+    addReplyLongLong(c,s->length);
+    addReplyBulkCString(c,"radix-tree-keys");
+    addReplyLongLong(c,raxSize(s->rax));
+    addReplyBulkCString(c,"radix-tree-nodes");
+    addReplyLongLong(c,s->rax->numnodes);
+    addReplyBulkCString(c,"last-generated-id");
+    addReplyStreamID(c,&s->last_id);
+
+    if (!full) {
+        /* XINFO STREAM <key> */
+
+        addReplyBulkCString(c,"groups");
+        addReplyLongLong(c,s->cgroups ? raxSize(s->cgroups) : 0);
+
+        /* To emit the first/last entry we use streamReplyWithRange(). */
+        int emitted;
+        streamID start, end;
+        start.ms = start.seq = 0;
+        end.ms = end.seq = UINT64_MAX;
+        addReplyBulkCString(c,"first-entry");
+        emitted = streamReplyWithRange(c,s,&start,&end,1,0,NULL,NULL,
+                                     STREAM_RWR_RAWENTRIES,NULL);
+        if (!emitted) addReplyNull(c);
+        addReplyBulkCString(c,"last-entry");
+        emitted = streamReplyWithRange(c,s,&start,&end,1,1,NULL,NULL,
+                                     STREAM_RWR_RAWENTRIES,NULL);
+        if (!emitted) addReplyNull(c);
+    } else {
+        /* XINFO STREAM <key> FULL [COUNT <count>] */
+
+        /* Stream entries */
+        addReplyBulkCString(c,"entries");
+        streamReplyWithRange(c,s,NULL,NULL,count,0,NULL,NULL,0,NULL);
+
+        /* Consumer groups */
+        addReplyBulkCString(c,"groups");
+        if (s->cgroups == NULL) {
+            addReplyArrayLen(c,0);
+        } else {
+            addReplyArrayLen(c,raxSize(s->cgroups));
+            raxIterator ri_cgroups;
+            raxStart(&ri_cgroups,s->cgroups);
+            raxSeek(&ri_cgroups,"^",NULL,0);
+            while(raxNext(&ri_cgroups)) {
+                streamCG *cg = ri_cgroups.data;
+                addReplyMapLen(c,5);
+
+                /* Name */
+                addReplyBulkCString(c,"name");
+                addReplyBulkCBuffer(c,ri_cgroups.key,ri_cgroups.key_len);
+
+                /* Last delivered ID */
+                addReplyBulkCString(c,"last-delivered-id");
+                addReplyStreamID(c,&cg->last_id);
+
+                /* Group PEL count */
+                addReplyBulkCString(c,"pel-count");
+                addReplyLongLong(c,raxSize(cg->pel));
+
+                /* Group PEL */
+                addReplyBulkCString(c,"pending");
+                long long arraylen_cg_pel = 0;
+                void *arrayptr_cg_pel = addReplyDeferredLen(c);
+                raxIterator ri_cg_pel;
+                raxStart(&ri_cg_pel,cg->pel);
+                raxSeek(&ri_cg_pel,"^",NULL,0);
+                while(raxNext(&ri_cg_pel) && (!count || arraylen_cg_pel < count)) {
+                    streamNACK *nack = ri_cg_pel.data;
+                    addReplyArrayLen(c,4);
+
+                    /* Entry ID. */
+                    streamID id;
+                    streamDecodeID(ri_cg_pel.key,&id);
+                    addReplyStreamID(c,&id);
+
+                    /* Consumer name. */
+                    addReplyBulkCBuffer(c,nack->consumer->name,
+                                        sdslen(nack->consumer->name));
+
+                    /* Last delivery. */
+                    addReplyLongLong(c,nack->delivery_time);
+
+                    /* Number of deliveries. */
+                    addReplyLongLong(c,nack->delivery_count);
+
+                    arraylen_cg_pel++;
+                }
+                setDeferredArrayLen(c,arrayptr_cg_pel,arraylen_cg_pel);
+                raxStop(&ri_cg_pel);
+
+                /* Consumers */
+                addReplyBulkCString(c,"consumers");
+                addReplyArrayLen(c,raxSize(cg->consumers));
+                raxIterator ri_consumers;
+                raxStart(&ri_consumers,cg->consumers);
+                raxSeek(&ri_consumers,"^",NULL,0);
+                while(raxNext(&ri_consumers)) {
+                    streamConsumer *consumer = ri_consumers.data;
+                    addReplyMapLen(c,4);
+
+                    /* Consumer name */
+                    addReplyBulkCString(c,"name");
+                    addReplyBulkCBuffer(c,consumer->name,sdslen(consumer->name));
+
+                    /* Seen-time */
+                    addReplyBulkCString(c,"seen-time");
+                    addReplyLongLong(c,consumer->seen_time);
+
+                    /* Consumer PEL count */
+                    addReplyBulkCString(c,"pel-count");
+                    addReplyLongLong(c,raxSize(consumer->pel));
+
+                    /* Consumer PEL */
+                    addReplyBulkCString(c,"pending");
+                    long long arraylen_cpel = 0;
+                    void *arrayptr_cpel = addReplyDeferredLen(c);
+                    raxIterator ri_cpel;
+                    raxStart(&ri_cpel,consumer->pel);
+                    raxSeek(&ri_cpel,"^",NULL,0);
+                    while(raxNext(&ri_cpel) && (!count || arraylen_cpel < count)) {
+                        streamNACK *nack = ri_cpel.data;
+                        addReplyArrayLen(c,3);
+
+                        /* Entry ID. */
+                        streamID id;
+                        streamDecodeID(ri_cpel.key,&id);
+                        addReplyStreamID(c,&id);
+
+                        /* Last delivery. */
+                        addReplyLongLong(c,nack->delivery_time);
+
+                        /* Number of deliveries. */
+                        addReplyLongLong(c,nack->delivery_count);
+
+                        arraylen_cpel++;
+                    }
+                    setDeferredArrayLen(c,arrayptr_cpel,arraylen_cpel);
+                    raxStop(&ri_cpel);
+                }
+                raxStop(&ri_consumers);
+            }
+            raxStop(&ri_cgroups);
+        }
+    }
+}
+
 /* XINFO CONSUMERS <key> <group>
  * XINFO GROUPS <key>
- * XINFO STREAM <key>
+ * XINFO STREAM <key> [FULL [COUNT <count>]]
  * XINFO HELP. */
 void xinfoCommand(client *c) {
     const char *help[] = {
-"CONSUMERS <key> <groupname>  -- Show consumer groups of group <groupname>.",
-"GROUPS <key>                 -- Show the stream consumer groups.",
-"STREAM <key>                 -- Show information about the stream.",
-"HELP                         -- Print this help.",
+"CONSUMERS <key> <groupname>         -- Show consumer groups of group <groupname>.",
+"GROUPS <key>                        -- Show the stream consumer groups.",
+"STREAM <key> [FULL [COUNT <count>]] -- Show information about the stream.",
+"HELP                                -- Print this help.",
 NULL
     };
     stream *s = NULL;
@@ -2578,36 +2762,10 @@ NULL
             addReplyStreamID(c,&cg->last_id);
         }
         raxStop(&ri);
-    } else if (!strcasecmp(opt,"STREAM") && c->argc == 3) {
-        /* XINFO STREAM <key> (or the alias XINFO <key>). */
-        addReplyMapLen(c,7);
-        addReplyBulkCString(c,"length");
-        addReplyLongLong(c,s->length);
-        addReplyBulkCString(c,"radix-tree-keys");
-        addReplyLongLong(c,raxSize(s->rax));
-        addReplyBulkCString(c,"radix-tree-nodes");
-        addReplyLongLong(c,s->rax->numnodes);
-        addReplyBulkCString(c,"groups");
-        addReplyLongLong(c,s->cgroups ? raxSize(s->cgroups) : 0);
-        addReplyBulkCString(c,"last-generated-id");
-        addReplyStreamID(c,&s->last_id);
-
-        /* To emit the first/last entry we us the streamReplyWithRange()
-         * API. */
-        int count;
-        streamID start, end;
-        start.ms = start.seq = 0;
-        end.ms = end.seq = UINT64_MAX;
-        addReplyBulkCString(c,"first-entry");
-        count = streamReplyWithRange(c,s,&start,&end,1,0,NULL,NULL,
-                                     STREAM_RWR_RAWENTRIES,NULL);
-        if (!count) addReplyNull(c);
-        addReplyBulkCString(c,"last-entry");
-        count = streamReplyWithRange(c,s,&start,&end,1,1,NULL,NULL,
-                                     STREAM_RWR_RAWENTRIES,NULL);
-        if (!count) addReplyNull(c);
+    } else if (!strcasecmp(opt,"STREAM")) {
+        /* XINFO STREAM <key> [FULL [COUNT <count>]]. */
+        xinfoReplyWithStreamInfo(c,s);
     } else {
         addReplySubcommandSyntaxError(c);
     }
 }
-
diff --git a/tests/unit/type/stream-cgroups.tcl b/tests/unit/type/stream-cgroups.tcl
index 04661707b..dfcd735f6 100644
--- a/tests/unit/type/stream-cgroups.tcl
+++ b/tests/unit/type/stream-cgroups.tcl
@@ -294,6 +294,40 @@ start_server {
         assert {[lindex $reply 0 3] == 2}
     }
 
+    test {XINFO FULL output} {
+        r del x
+        r XADD x 100 a 1
+        r XADD x 101 b 1
+        r XADD x 102 c 1
+        r XADD x 103 e 1
+        r XADD x 104 f 1
+        r XGROUP CREATE x g1 0
+        r XGROUP CREATE x g2 0
+        r XREADGROUP GROUP g1 Alice COUNT 1 STREAMS x >
+        r XREADGROUP GROUP g1 Bob COUNT 1 STREAMS x >
+        r XREADGROUP GROUP g1 Bob NOACK COUNT 1 STREAMS x >
+        r XREADGROUP GROUP g2 Charlie COUNT 4 STREAMS x >
+        r XDEL x 103
+
+        set reply [r XINFO STREAM x FULL]
+        assert_equal [llength $reply] 12
+        assert_equal [lindex $reply 1] 4 ;# stream length
+        assert_equal [lindex $reply 9] "{100-0 {a 1}} {101-0 {b 1}} {102-0 {c 1}} {104-0 {f 1}}" ;# entries
+        assert_equal [lindex $reply 11 0 1] "g1" ;# first group name
+        assert_equal [lindex $reply 11 0 7 0 0] "100-0" ;# first entry in group's PEL
+        assert_equal [lindex $reply 11 0 9 0 1] "Alice" ;# first consumer
+        assert_equal [lindex $reply 11 0 9 0 7 0 0] "100-0" ;# first entry in first consumer's PEL
+        assert_equal [lindex $reply 11 1 1] "g2" ;# second group name
+        assert_equal [lindex $reply 11 1 9 0 1] "Charlie" ;# first consumer
+        assert_equal [lindex $reply 11 1 9 0 7 0 0] "100-0" ;# first entry in first consumer's PEL
+        assert_equal [lindex $reply 11 1 9 0 7 1 0] "101-0" ;# second entry in first consumer's PEL
+
+        set reply [r XINFO STREAM x FULL COUNT 1]
+        assert_equal [llength $reply] 12
+        assert_equal [lindex $reply 1] 4
+        assert_equal [lindex $reply 9] "{100-0 {a 1}}"
+    }
+
     start_server {} {
         set master [srv -1 client]
         set master_host [srv -1 host]

From 47b8a7f9b881c23598c3508e597f9c0d1e997ae2 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Tue, 28 Apr 2020 16:40:15 +0200
Subject: [PATCH 106/186] Fix create-cluster BIN_PATH.

---
 utils/create-cluster/create-cluster | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/utils/create-cluster/create-cluster b/utils/create-cluster/create-cluster
index 4db0a6619..931f6f521 100755
--- a/utils/create-cluster/create-cluster
+++ b/utils/create-cluster/create-cluster
@@ -1,7 +1,7 @@
 #!/bin/bash
 
 # Settings
-BIN_PATH="../../"
+BIN_PATH="../../src/"
 CLUSTER_HOST=127.0.0.1
 PORT=30000
 TIMEOUT=2000

From ee627bb667c6fb08af64fb4520631e8530013ca2 Mon Sep 17 00:00:00 2001
From: srzhao <srzhao@sysnew.com>
Date: Tue, 26 Nov 2019 10:43:57 +0800
Subject: [PATCH 107/186] fix pipelined WAIT performance issue.

If client gets blocked again in `processUnblockedClients`, redis will not send
`REPLCONF GETACK *` to slaves untill next eventloop, so the client will be
blocked for 100ms by default(10hz) if no other file event fired.

move server.get_ack_from_slaves sinppet after `processUnblockedClients`, so
that both the first WAIT command that puts client in blocked context and the
following WAIT command processed in processUnblockedClients would trigger
redis-sever to send `REPLCONF GETACK *`, so that the eventloop would get
`REPLCONG ACK <reploffset>` from slaves and unblocked ASAP.
---
 src/server.c | 26 +++++++++++++-------------
 1 file changed, 13 insertions(+), 13 deletions(-)

diff --git a/src/server.c b/src/server.c
index ab6280121..150476276 100644
--- a/src/server.c
+++ b/src/server.c
@@ -2110,6 +2110,19 @@ void beforeSleep(struct aeEventLoop *eventLoop) {
     if (server.active_expire_enabled && server.masterhost == NULL)
         activeExpireCycle(ACTIVE_EXPIRE_CYCLE_FAST);
 
+    /* Unblock all the clients blocked for synchronous replication
+     * in WAIT. */
+    if (listLength(server.clients_waiting_acks))
+        processClientsWaitingReplicas();
+
+    /* Check if there are clients unblocked by modules that implement
+     * blocking commands. */
+    if (moduleCount()) moduleHandleBlockedClients();
+
+    /* Try to process pending commands for clients that were just unblocked. */
+    if (listLength(server.unblocked_clients))
+        processUnblockedClients();
+
     /* Send all the slaves an ACK request if at least one client blocked
      * during the previous event loop iteration. */
     if (server.get_ack_from_slaves) {
@@ -2125,19 +2138,6 @@ void beforeSleep(struct aeEventLoop *eventLoop) {
         server.get_ack_from_slaves = 0;
     }
 
-    /* Unblock all the clients blocked for synchronous replication
-     * in WAIT. */
-    if (listLength(server.clients_waiting_acks))
-        processClientsWaitingReplicas();
-
-    /* Check if there are clients unblocked by modules that implement
-     * blocking commands. */
-    if (moduleCount()) moduleHandleBlockedClients();
-
-    /* Try to process pending commands for clients that were just unblocked. */
-    if (listLength(server.unblocked_clients))
-        processUnblockedClients();
-
     /* Send the invalidation messages to clients participating to the
      * client side caching protocol in broadcasting (BCAST) mode. */
     trackingBroadcastInvalidationMessages();

From 606134f9de09299fdd704bab5df7dd9799e35959 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Wed, 29 Apr 2020 11:16:30 +0200
Subject: [PATCH 108/186] Comment clearly why we moved some code in #6623.

---
 src/server.c | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/server.c b/src/server.c
index 150476276..659604ef3 100644
--- a/src/server.c
+++ b/src/server.c
@@ -2124,7 +2124,10 @@ void beforeSleep(struct aeEventLoop *eventLoop) {
         processUnblockedClients();
 
     /* Send all the slaves an ACK request if at least one client blocked
-     * during the previous event loop iteration. */
+     * during the previous event loop iteration. Note that we do this after
+     * processUnblockedClients(), so if there are multiple pipelined WAITs
+     * and the just unblocked WAIT gets blocked again, we don't have to wait
+     * a server cron cycle in absence of other event loop events. See #6623. */
     if (server.get_ack_from_slaves) {
         robj *argv[3];
 

From aab74b7151c8ce646f24c65405f3fd5f58c03577 Mon Sep 17 00:00:00 2001
From: Guy Benoish <guy.benoish@redislabs.com>
Date: Tue, 28 Apr 2020 17:58:25 +0300
Subject: [PATCH 109/186] XINFO STREAM FULL should have a default COUNT of 10

---
 src/t_stream.c | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/src/t_stream.c b/src/t_stream.c
index d6a5e0009..5c1b9a523 100644
--- a/src/t_stream.c
+++ b/src/t_stream.c
@@ -2493,7 +2493,7 @@ void xtrimCommand(client *c) {
  * Handles the variants of XINFO STREAM */
 void xinfoReplyWithStreamInfo(client *c, stream *s) {
     int full = 1;
-    long long count = 0;
+    long long count = 10; /* Default COUNT is 10 so we don't block the server */
     robj **optv = c->argv + 3; /* Options start after XINFO STREAM <key> */
     int optc = c->argc - 3;
 
@@ -2521,7 +2521,7 @@ void xinfoReplyWithStreamInfo(client *c, stream *s) {
             }
             if (getLongLongFromObjectOrReply(c,optv[2],&count,NULL) == C_ERR)
                 return;
-            if (count < 0) count = 0;
+            if (count < 0) count = 10;
         }
     }
 
@@ -2548,11 +2548,11 @@ void xinfoReplyWithStreamInfo(client *c, stream *s) {
         end.ms = end.seq = UINT64_MAX;
         addReplyBulkCString(c,"first-entry");
         emitted = streamReplyWithRange(c,s,&start,&end,1,0,NULL,NULL,
-                                     STREAM_RWR_RAWENTRIES,NULL);
+                                       STREAM_RWR_RAWENTRIES,NULL);
         if (!emitted) addReplyNull(c);
         addReplyBulkCString(c,"last-entry");
         emitted = streamReplyWithRange(c,s,&start,&end,1,1,NULL,NULL,
-                                     STREAM_RWR_RAWENTRIES,NULL);
+                                       STREAM_RWR_RAWENTRIES,NULL);
         if (!emitted) addReplyNull(c);
     } else {
         /* XINFO STREAM <key> FULL [COUNT <count>] */
@@ -2682,6 +2682,10 @@ void xinfoCommand(client *c) {
 "CONSUMERS <key> <groupname>         -- Show consumer groups of group <groupname>.",
 "GROUPS <key>                        -- Show the stream consumer groups.",
 "STREAM <key> [FULL [COUNT <count>]] -- Show information about the stream.",
+"                                       FULL will return the full state of the stream,",
+"                                            including all entries, groups, consumers and PELs.",
+"                                            It's possible to show only the first stream/PEL entries",
+"                                            by using the COUNT modifier (Default is 10)",
 "HELP                                -- Print this help.",
 NULL
     };

From c163d4addc1cb693e702fae58c1dfa4c1c589164 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Wed, 29 Apr 2020 12:37:12 +0200
Subject: [PATCH 110/186] redis-cli: try to make clusterManagerFixOpenSlot()
 more readable.

Also improve the message to make clear that there is no *clear* owner,
not that there is no owner at all.
---
 src/redis-cli.c | 31 +++++++++++++++++++++++++------
 1 file changed, 25 insertions(+), 6 deletions(-)

diff --git a/src/redis-cli.c b/src/redis-cli.c
index 72480d08c..469dbb0ff 100644
--- a/src/redis-cli.c
+++ b/src/redis-cli.c
@@ -4596,20 +4596,26 @@ static int clusterManagerFixOpenSlot(int slot) {
     /* Try to obtain the current slot owner, according to the current
      * nodes configuration. */
     int success = 1;
-    list *owners = listCreate();
+    list *owners = listCreate();    /* List of nodes claiming some ownership.
+                                       it could be stating in the configuration
+                                       to have the node ownership, or just
+                                       holding keys for such slot. */
     list *migrating = listCreate();
     list *importing = listCreate();
     sds migrating_str = sdsempty();
     sds importing_str = sdsempty();
-    clusterManagerNode *owner = NULL;
+    clusterManagerNode *owner = NULL; /* The obvious slot owner if any. */
+
+    /* Iterate all the nodes, looking for potential owners of this slot. */
     listIter li;
     listNode *ln;
     listRewind(cluster_manager.nodes, &li);
     while ((ln = listNext(&li)) != NULL) {
         clusterManagerNode *n = ln->value;
         if (n->flags & CLUSTER_MANAGER_FLAG_SLAVE) continue;
-        if (n->slots[slot]) listAddNodeTail(owners, n);
-        else {
+        if (n->slots[slot]) {
+            listAddNodeTail(owners, n);
+        } else {
             redisReply *r = CLUSTER_MANAGER_COMMAND(n,
                 "CLUSTER COUNTKEYSINSLOT %d", slot);
             success = clusterManagerCheckRedisReply(n, r, NULL);
@@ -4623,7 +4629,14 @@ static int clusterManagerFixOpenSlot(int slot) {
             if (!success) goto cleanup;
         }
     }
+
+    /* If we have only a single potential owner for this slot,
+     * set it as "owner". */
     if (listLength(owners) == 1) owner = listFirst(owners)->value;
+
+    /* Scan the list of nodes again, in order to populate the
+     * list of nodes in importing or migrating state for
+     * this slot. */
     listRewind(cluster_manager.nodes, &li);
     while ((ln = listNext(&li)) != NULL) {
         clusterManagerNode *n = ln->value;
@@ -4655,6 +4668,7 @@ static int clusterManagerFixOpenSlot(int slot) {
                 }
             }
         }
+
         /* If the node is neither migrating nor importing and it's not
          * the owner, then is added to the importing list in case
          * it has keys in the slot. */
@@ -4679,11 +4693,12 @@ static int clusterManagerFixOpenSlot(int slot) {
         printf("Set as migrating in: %s\n", migrating_str);
     if (sdslen(importing_str) > 0)
         printf("Set as importing in: %s\n", importing_str);
+
     /* If there is no slot owner, set as owner the node with the biggest
      * number of keys, among the set of migrating / importing nodes. */
     if (owner == NULL) {
-        clusterManagerLogInfo(">>> Nobody claims ownership, "
-                              "selecting an owner...\n");
+        clusterManagerLogInfo(">>> No single clear owner for the slot, "
+                              "selecting an owner by # of keys...\n");
         owner = clusterManagerGetNodeWithMostKeysInSlot(cluster_manager.nodes,
                                                         slot, NULL);
         // If we still don't have an owner, we can't fix it.
@@ -4714,6 +4729,7 @@ static int clusterManagerFixOpenSlot(int slot) {
         clusterManagerRemoveNodeFromList(migrating, owner);
         clusterManagerRemoveNodeFromList(importing, owner);
     }
+
     /* If there are multiple owners of the slot, we need to fix it
      * so that a single node is the owner and all the other nodes
      * are in importing state. Later the fix can be handled by one
@@ -4746,6 +4762,7 @@ static int clusterManagerFixOpenSlot(int slot) {
         }
     }
     int move_opts = CLUSTER_MANAGER_OPT_VERBOSE;
+
     /* Case 1: The slot is in migrating state in one node, and in
      *         importing state in 1 node. That's trivial to address. */
     if (listLength(migrating) == 1 && listLength(importing) == 1) {
@@ -4757,6 +4774,7 @@ static int clusterManagerFixOpenSlot(int slot) {
         move_opts |= CLUSTER_MANAGER_OPT_UPDATE;
         success = clusterManagerMoveSlot(src, dst, slot, move_opts, NULL);
     }
+
     /* Case 2: There are multiple nodes that claim the slot as importing,
      * they probably got keys about the slot after a restart so opened
      * the slot. In this case we just move all the keys to the owner
@@ -4787,6 +4805,7 @@ static int clusterManagerFixOpenSlot(int slot) {
             if (!success) goto cleanup;
         }
     }
+
     /* Case 3: The slot is in migrating state in one node but multiple
      * other nodes claim to be in importing state and don't have any key in
      * the slot. We search for the importing node having the same ID as

From 5b59d9c5d793c9d82ab053c8efb4ec364e857873 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Wed, 29 Apr 2020 16:28:16 +0200
Subject: [PATCH 111/186] redis-cli: simplify cluster nodes coverage display.

---
 src/redis-cli.c | 27 ++++++++++-----------------
 1 file changed, 10 insertions(+), 17 deletions(-)

diff --git a/src/redis-cli.c b/src/redis-cli.c
index 469dbb0ff..82a46216c 100644
--- a/src/redis-cli.c
+++ b/src/redis-cli.c
@@ -4295,17 +4295,18 @@ static int clusterManagerGetCoveredSlots(char *all_slots) {
 }
 
 static void clusterManagerPrintSlotsList(list *slots) {
+    clusterManagerNode n = {0};
     listIter li;
     listNode *ln;
     listRewind(slots, &li);
-    sds first = NULL;
     while ((ln = listNext(&li)) != NULL) {
-        sds slot = ln->value;
-        if (!first) first = slot;
-        else printf(", ");
-        printf("%s", slot);
+        int slot = atoi(ln->value);
+        if (slot >= 0 && slot < CLUSTER_MANAGER_SLOTS)
+            n.slots[slot] = 1;
     }
-    printf("\n");
+    sds nodeslist = clusterManagerNodeSlotsString(&n);
+    printf("%s\n", nodeslist);
+    sdsfree(nodeslist);
 }
 
 /* Return the node, among 'nodes' with the greatest number of keys
@@ -4398,15 +4399,10 @@ static int clusterManagerFixSlotsCoverage(char *all_slots) {
     int i, fixed = 0;
     list *none = NULL, *single = NULL, *multi = NULL;
     clusterManagerLogInfo(">>> Fixing slots coverage...\n");
-    printf("List of not covered slots: \n");
-    int uncovered_count = 0;
-    sds log = sdsempty();
     for (i = 0; i < CLUSTER_MANAGER_SLOTS; i++) {
         int covered = all_slots[i];
         if (!covered) {
-            sds key = sdsfromlonglong((long long) i);
-            if (uncovered_count++ > 0) printf(",");
-            printf("%s", (char *) key);
+            sds slot = sdsfromlonglong((long long) i);
             list *slot_nodes = listCreate();
             sds slot_nodes_str = sdsempty();
             listIter li;
@@ -4433,13 +4429,11 @@ static int clusterManagerFixSlotsCoverage(char *all_slots) {
                 }
                 freeReplyObject(reply);
             }
-            log = sdscatfmt(log, "\nSlot %S has keys in %u nodes: %S",
-                            key, listLength(slot_nodes), slot_nodes_str);
             sdsfree(slot_nodes_str);
-            dictAdd(clusterManagerUncoveredSlots, key, slot_nodes);
+            dictAdd(clusterManagerUncoveredSlots, slot, slot_nodes);
         }
     }
-    printf("\n%s\n", log);
+
     /* For every slot, take action depending on the actual condition:
      * 1) No node has keys for this slot.
      * 2) A single node has keys for this slot.
@@ -4581,7 +4575,6 @@ static int clusterManagerFixSlotsCoverage(char *all_slots) {
         }
     }
 cleanup:
-    sdsfree(log);
     if (none) listRelease(none);
     if (single) listRelease(single);
     if (multi) listRelease(multi);

From 96dd5fc932ea4f3ae39e7b8f249769bae23fe3cd Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Wed, 29 Apr 2020 16:57:06 +0200
Subject: [PATCH 112/186] redis-cli: safer cluster fix with unreachalbe
 masters.

---
 src/redis-cli.c | 27 ++++++++++++++++++++++++++-
 1 file changed, 26 insertions(+), 1 deletion(-)

diff --git a/src/redis-cli.c b/src/redis-cli.c
index 82a46216c..ec0f58f5f 100644
--- a/src/redis-cli.c
+++ b/src/redis-cli.c
@@ -123,6 +123,7 @@
 #define CLUSTER_MANAGER_CMD_FLAG_COPY           1 << 7
 #define CLUSTER_MANAGER_CMD_FLAG_COLOR          1 << 8
 #define CLUSTER_MANAGER_CMD_FLAG_CHECK_OWNERS   1 << 9
+#define CLUSTER_MANAGER_CMD_FLAG_FIX_WITH_UNREACHABLE_MASTERS 1 << 10
 
 #define CLUSTER_MANAGER_OPT_GETFRIENDS  1 << 0
 #define CLUSTER_MANAGER_OPT_COLD        1 << 1
@@ -1599,6 +1600,9 @@ static int parseOptions(int argc, char **argv) {
         } else if (!strcmp(argv[i],"--cluster-search-multiple-owners")) {
             config.cluster_manager_command.flags |=
                 CLUSTER_MANAGER_CMD_FLAG_CHECK_OWNERS;
+        } else if (!strcmp(argv[i],"--cluster-fix-with-unreachable-masters")) {
+            config.cluster_manager_command.flags |=
+                CLUSTER_MANAGER_CMD_FLAG_FIX_WITH_UNREACHABLE_MASTERS;
 #ifdef USE_OPENSSL
         } else if (!strcmp(argv[i],"--tls")) {
             config.tls = 1;
@@ -2146,6 +2150,7 @@ static int evalMode(int argc, char **argv) {
 static struct clusterManager {
     list *nodes;    /* List of nodes in the configuration. */
     list *errors;
+    int unreachable_masters;    /* Masters we are not able to reach. */
 } cluster_manager;
 
 /* Used by clusterManagerFixSlotsCoverage */
@@ -2288,7 +2293,7 @@ clusterManagerCommandDef clusterManagerCommands[] = {
      "search-multiple-owners"},
     {"info", clusterManagerCommandInfo, -1, "host:port", NULL},
     {"fix", clusterManagerCommandFix, -1, "host:port",
-     "search-multiple-owners"},
+     "search-multiple-owners,fix-with-unreachable-masters"},
     {"reshard", clusterManagerCommandReshard, -1, "host:port",
      "from <arg>,to <arg>,slots <arg>,yes,timeout <arg>,pipeline <arg>,"
      "replace"},
@@ -4013,7 +4018,9 @@ static int clusterManagerLoadInfoFromNode(clusterManagerNode *node, int opts) {
                 if (friend->flags & (CLUSTER_MANAGER_FLAG_NOADDR |
                                      CLUSTER_MANAGER_FLAG_DISCONNECT |
                                      CLUSTER_MANAGER_FLAG_FAIL))
+                {
                     goto invalid_friend;
+                }
                 listAddNodeTail(cluster_manager.nodes, friend);
             } else {
                 clusterManagerLogErr("[ERR] Unable to load info for "
@@ -4023,6 +4030,8 @@ static int clusterManagerLoadInfoFromNode(clusterManagerNode *node, int opts) {
             }
             continue;
 invalid_friend:
+            if (!(friend->flags & CLUSTER_MANAGER_FLAG_SLAVE))
+                cluster_manager.unreachable_masters++;
             freeClusterManagerNode(friend);
         }
         listRelease(node->friends);
@@ -4396,6 +4405,14 @@ static clusterManagerNode *clusterManagerNodeMasterRandom() {
 }
 
 static int clusterManagerFixSlotsCoverage(char *all_slots) {
+    int force_fix = config.cluster_manager_command.flags &
+                    CLUSTER_MANAGER_CMD_FLAG_FIX_WITH_UNREACHABLE_MASTERS;
+
+    if (cluster_manager.unreachable_masters > 0 && !force_fix) {
+        clusterManagerLogWarn("*** Fixing slots coverage with %d unreachable masters is dangerous: redis-cli will assume that slots about masters that are not reachable are not covered, and will try to reassign them to the reachable nodes. This can cause data loss and is rarely what you want to do. If you really want to proceed use the --cluster-fix-with-unreachable-masters option.\n", cluster_manager.unreachable_masters);
+        exit(1);
+    }
+
     int i, fixed = 0;
     list *none = NULL, *single = NULL, *multi = NULL;
     clusterManagerLogInfo(">>> Fixing slots coverage...\n");
@@ -4585,6 +4602,14 @@ cleanup:
  * more nodes. This function fixes this condition by migrating keys where
  * it seems more sensible. */
 static int clusterManagerFixOpenSlot(int slot) {
+    int force_fix = config.cluster_manager_command.flags &
+                    CLUSTER_MANAGER_CMD_FLAG_FIX_WITH_UNREACHABLE_MASTERS;
+
+    if (cluster_manager.unreachable_masters > 0 && !force_fix) {
+        clusterManagerLogWarn("*** Fixing open slots with %d unreachable masters is dangerous: redis-cli will assume that slots about masters that are not reachable are not covered, and will try to reassign them to the reachable nodes. This can cause data loss and is rarely what you want to do. If you really want to proceed use the --cluster-fix-with-unreachable-masters option.\n", cluster_manager.unreachable_masters);
+        exit(1);
+    }
+
     clusterManagerLogInfo(">>> Fixing open slot %d\n", slot);
     /* Try to obtain the current slot owner, according to the current
      * nodes configuration. */

From d56f058c089dc255b31e698c3f4f08ca3a8cb627 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Wed, 29 Apr 2020 18:49:42 +0200
Subject: [PATCH 113/186] Fix tracking table max keys option in redis.conf.

---
 redis.conf | 21 ++++++++++++---------
 1 file changed, 12 insertions(+), 9 deletions(-)

diff --git a/redis.conf b/redis.conf
index fab0a5898..6cec636bc 100644
--- a/redis.conf
+++ b/redis.conf
@@ -626,20 +626,23 @@ replica-priority 100
 # to track the keys fetched by many clients.
 #
 # For this reason it is possible to configure a maximum fill value for the
-# invalidation table. By default it is set to 10%, and once this limit is
-# reached, Redis will start to evict caching slots in the invalidation table
-# even if keys are not modified, just to reclaim memory: this will in turn
+# invalidation table. By default it is set to 1M of keys, and once this limit
+# is reached, Redis will start to evict keys in the invalidation table
+# even if they were not modified, just to reclaim memory: this will in turn
 # force the clients to invalidate the cached values. Basically the table
-# maximum fill rate is a trade off between the memory you want to spend server
+# maximum size is a trade off between the memory you want to spend server
 # side to track information about who cached what, and the ability of clients
 # to retain cached objects in memory.
 #
-# If you set the value to 0, it means there are no limits, and all the 16
-# millions of caching slots can be used at the same time. In the "stats"
-# INFO section, you can find information about the amount of caching slots
-# used at every given moment.
+# If you set the value to 0, it means there are no limits, and Redis will
+# retain as many keys as needed in the invalidation table.
+# In the "stats" INFO section, you can find information about the number of
+# keys in the invalidation table at every given moment.
 #
-# tracking-table-max-fill 10
+# Note: when key tracking is used in broadcasting mode, no memory is used
+# in the server side so this setting is useless.
+#
+# tracking-table-max-keys 1000000
 
 ################################## SECURITY ###################################
 

From e1ee1a49d62b8637b6ba27362a15cc8b03d66d1c Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Thu, 30 Apr 2020 09:58:06 +0200
Subject: [PATCH 114/186] CLIENT KILL USER <username>.

---
 src/networking.c | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/src/networking.c b/src/networking.c
index 7ffa99eb1..767206ab9 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -2101,6 +2101,7 @@ void clientCommand(client *c) {
 "KILL <option> <value> [option value ...] -- Kill connections. Options are:",
 "     ADDR <ip:port>                      -- Kill connection made from <ip:port>",
 "     TYPE (normal|master|replica|pubsub) -- Kill connections by type.",
+"     USER <username>   -- Kill connections authenticated with such user.",
 "     SKIPME (yes|no)   -- Skip killing current connection (default: yes).",
 "LIST [options ...]     -- Return information about client connections. Options:",
 "     TYPE (normal|master|replica|pubsub) -- Return clients of specified type.",
@@ -2151,6 +2152,7 @@ NULL
         /* CLIENT KILL <ip:port>
          * CLIENT KILL <option> [value] ... <option> [value] */
         char *addr = NULL;
+        user *user = NULL;
         int type = -1;
         uint64_t id = 0;
         int skipme = 1;
@@ -2182,6 +2184,14 @@ NULL
                     }
                 } else if (!strcasecmp(c->argv[i]->ptr,"addr") && moreargs) {
                     addr = c->argv[i+1]->ptr;
+                } else if (!strcasecmp(c->argv[i]->ptr,"user") && moreargs) {
+                    user = ACLGetUserByName(c->argv[i+1]->ptr,
+                                            sdslen(c->argv[i+1]->ptr));
+                    if (user == NULL) {
+                        addReplyErrorFormat(c,"No such user '%s'",
+                            (char*) c->argv[i+1]->ptr);
+                        return;
+                    }
                 } else if (!strcasecmp(c->argv[i]->ptr,"skipme") && moreargs) {
                     if (!strcasecmp(c->argv[i+1]->ptr,"yes")) {
                         skipme = 1;
@@ -2209,6 +2219,7 @@ NULL
             if (addr && strcmp(getClientPeerId(client),addr) != 0) continue;
             if (type != -1 && getClientType(client) != type) continue;
             if (id != 0 && client->id != id) continue;
+            if (user && client->user != user) continue;
             if (c == client && skipme) continue;
 
             /* Kill it. */

From 7be21139aab7ebe813727ae4a6be8955dd2eac0b Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Thu, 30 Apr 2020 10:14:15 +0200
Subject: [PATCH 115/186] MIGRATE AUTH2 for ACL support.

---
 src/cluster.c | 24 +++++++++++++++++++-----
 1 file changed, 19 insertions(+), 5 deletions(-)

diff --git a/src/cluster.c b/src/cluster.c
index 49a410d54..d8f83f08d 100644
--- a/src/cluster.c
+++ b/src/cluster.c
@@ -5101,15 +5101,17 @@ void migrateCloseTimedoutSockets(void) {
     dictReleaseIterator(di);
 }
 
-/* MIGRATE host port key dbid timeout [COPY | REPLACE | AUTH password]
+/* MIGRATE host port key dbid timeout [COPY | REPLACE | AUTH password |
+ *         AUTH2 username password]
  *
  * On in the multiple keys form:
  *
- * MIGRATE host port "" dbid timeout [COPY | REPLACE | AUTH password] KEYS key1
- * key2 ... keyN */
+ * MIGRATE host port "" dbid timeout [COPY | REPLACE | AUTH password |
+ *         AUTH2 username password] KEYS key1 key2 ... keyN */
 void migrateCommand(client *c) {
     migrateCachedSocket *cs;
     int copy = 0, replace = 0, j;
+    char *username = NULL;
     char *password = NULL;
     long timeout;
     long dbid;
@@ -5127,7 +5129,7 @@ void migrateCommand(client *c) {
 
     /* Parse additional options */
     for (j = 6; j < c->argc; j++) {
-        int moreargs = j < c->argc-1;
+        int moreargs = (c->argc-1) - j;
         if (!strcasecmp(c->argv[j]->ptr,"copy")) {
             copy = 1;
         } else if (!strcasecmp(c->argv[j]->ptr,"replace")) {
@@ -5139,6 +5141,13 @@ void migrateCommand(client *c) {
             }
             j++;
             password = c->argv[j]->ptr;
+        } else if (!strcasecmp(c->argv[j]->ptr,"auth2")) {
+            if (moreargs < 2) {
+                addReply(c,shared.syntaxerr);
+                return;
+            }
+            username = c->argv[++j]->ptr;
+            password = c->argv[++j]->ptr;
         } else if (!strcasecmp(c->argv[j]->ptr,"keys")) {
             if (sdslen(c->argv[3]->ptr) != 0) {
                 addReplyError(c,
@@ -5199,8 +5208,13 @@ try_again:
 
     /* Authentication */
     if (password) {
-        serverAssertWithInfo(c,NULL,rioWriteBulkCount(&cmd,'*',2));
+        int arity = username ? 3 : 2;
+        serverAssertWithInfo(c,NULL,rioWriteBulkCount(&cmd,'*',arity));
         serverAssertWithInfo(c,NULL,rioWriteBulkString(&cmd,"AUTH",4));
+        if (username) {
+            serverAssertWithInfo(c,NULL,rioWriteBulkString(&cmd,username,
+                                 sdslen(username)));
+        }
         serverAssertWithInfo(c,NULL,rioWriteBulkString(&cmd,password,
             sdslen(password)));
     }

From 70287bbc942acbd0b73b1725f381a919112e273c Mon Sep 17 00:00:00 2001
From: "zhaozhao.zz" <zhaozhao.zz@alibaba-inc.com>
Date: Wed, 27 Feb 2019 19:38:03 +0800
Subject: [PATCH 116/186] lazyfree & eviction: record latency generated by
 lazyfree eviction

1. add eviction-lazyfree monitor
2. put eviction-del & eviction-lazyfree into eviction-cycle
   that means eviction-cycle contains all the latency in
   the eviction cycle including del and lazyfree
3. use getMaxmemoryState to check if we can break in lazyfree-evict
---
 src/evict.c | 31 ++++++++++++++++++-------------
 1 file changed, 18 insertions(+), 13 deletions(-)

diff --git a/src/evict.c b/src/evict.c
index a829f1f99..0755acc0e 100644
--- a/src/evict.c
+++ b/src/evict.c
@@ -450,9 +450,10 @@ int freeMemoryIfNeeded(void) {
     if (server.masterhost && server.repl_slave_ignore_maxmemory) return C_OK;
 
     size_t mem_reported, mem_tofree, mem_freed;
-    mstime_t latency, eviction_latency;
+    mstime_t latency, eviction_latency, lazyfree_latency;
     long long delta;
     int slaves = listLength(server.slaves);
+    int result = C_ERR;
 
     /* When clients are paused the dataset should be static not just from the
      * POV of clients not being able to write, but also from the POV of
@@ -463,10 +464,10 @@ int freeMemoryIfNeeded(void) {
 
     mem_freed = 0;
 
+    latencyStartMonitor(latency);
     if (server.maxmemory_policy == MAXMEMORY_NO_EVICTION)
         goto cant_free; /* We need to free memory, but policy forbids. */
 
-    latencyStartMonitor(latency);
     while (mem_freed < mem_tofree) {
         int j, k, i;
         static unsigned int next_db = 0;
@@ -572,7 +573,6 @@ int freeMemoryIfNeeded(void) {
             signalModifiedKey(NULL,db,keyobj);
             latencyEndMonitor(eviction_latency);
             latencyAddSampleIfNeeded("eviction-del",eviction_latency);
-            latencyRemoveNestedEvent(latency,eviction_latency);
             delta -= (long long) zmalloc_used_memory();
             mem_freed += delta;
             server.stat_evictedkeys++;
@@ -601,25 +601,30 @@ int freeMemoryIfNeeded(void) {
                 }
             }
         } else {
-            latencyEndMonitor(latency);
-            latencyAddSampleIfNeeded("eviction-cycle",latency);
             goto cant_free; /* nothing to free... */
         }
     }
-    latencyEndMonitor(latency);
-    latencyAddSampleIfNeeded("eviction-cycle",latency);
-    return C_OK;
+    result = C_OK;
 
 cant_free:
     /* We are here if we are not able to reclaim memory. There is only one
      * last thing we can try: check if the lazyfree thread has jobs in queue
      * and wait... */
-    while(bioPendingJobsOfType(BIO_LAZY_FREE)) {
-        if (((mem_reported - zmalloc_used_memory()) + mem_freed) >= mem_tofree)
-            break;
-        usleep(1000);
+    if (result != C_OK) {
+        latencyStartMonitor(lazyfree_latency);
+        while(bioPendingJobsOfType(BIO_LAZY_FREE)) {
+            if (getMaxmemoryState(NULL,NULL,NULL,NULL) == C_OK) {
+                result = C_OK;
+                break;
+            }
+            usleep(1000);
+        }
+        latencyEndMonitor(lazyfree_latency);
+        latencyAddSampleIfNeeded("eviction-lazyfree",lazyfree_latency);
     }
-    return C_ERR;
+    latencyEndMonitor(latency);
+    latencyAddSampleIfNeeded("eviction-cycle",latency);
+    return result;
 }
 
 /* This is a wrapper for freeMemoryIfNeeded() that only really calls the

From 455d8a05c1aa5b415c768bf160934a38b14a1f4e Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Thu, 30 Apr 2020 13:00:37 +0200
Subject: [PATCH 117/186] redis-cli command help updated.

---
 src/help.h | 190 ++++++++++++++++++++++++++++++++++++++++++++++-------
 1 file changed, 165 insertions(+), 25 deletions(-)

diff --git a/src/help.h b/src/help.h
index 184d76724..6aa875541 100644
--- a/src/help.h
+++ b/src/help.h
@@ -28,6 +28,56 @@ struct commandHelp {
   int group;
   char *since;
 } commandHelp[] = {
+    { "ACL CAT",
+    "[categoryname]",
+    "List the ACL categories or the commands inside a category",
+    9,
+    "6.0.0" },
+    { "ACL DELUSER",
+    "username [username ...]",
+    "Remove the specified ACL users and the associated rules",
+    9,
+    "6.0.0" },
+    { "ACL GENPASS",
+    "[bits]",
+    "Generate a pseudorandom secure password to use for ACL users",
+    9,
+    "6.0.0" },
+    { "ACL LIST",
+    "-",
+    "List the current ACL rules in ACL config file format",
+    9,
+    "6.0.0" },
+    { "ACL LOAD",
+    "-",
+    "Reload the ACLs from the configured ACL file",
+    9,
+    "6.0.0" },
+    { "ACL LOG",
+    "[count or RESET]",
+    "List latest events denied because of ACLs in place",
+    9,
+    "6.0.0" },
+    { "ACL SAVE",
+    "-",
+    "Save the current ACL rules in the configured ACL file",
+    9,
+    "6.0.0" },
+    { "ACL SETUSER",
+    "rule [rule ...]",
+    "Modify or create the rules for a specific ACL user",
+    9,
+    "6.0.0" },
+    { "ACL USERS",
+    "-",
+    "List the username of all the configured ACL rules",
+    9,
+    "6.0.0" },
+    { "ACL WHOAMI",
+    "-",
+    "Return the name of the user associated to the current connection",
+    9,
+    "6.0.0" },
     { "APPEND",
     "key value",
     "Append a value to a key",
@@ -44,7 +94,7 @@ struct commandHelp {
     9,
     "1.0.0" },
     { "BGSAVE",
-    "-",
+    "[SCHEDULE]",
     "Asynchronously save the dataset to disk",
     9,
     "1.0.0" },
@@ -80,7 +130,7 @@ struct commandHelp {
     "2.0.0" },
     { "BRPOPLPUSH",
     "source destination timeout",
-    "Pop a value from a list, push it to another list and return it; or block until one is available",
+    "Pop an element from a list, push it to another list and return it; or block until one is available",
     2,
     "2.2.0" },
     { "BZPOPMAX",
@@ -93,11 +143,21 @@ struct commandHelp {
     "Remove and return the member with the lowest score from one or more sorted sets, or block until one is available",
     4,
     "5.0.0" },
+    { "CLIENT CACHING",
+    "YES|NO",
+    "Instruct the server about tracking or not keys in the next request",
+    9,
+    "6.0.0" },
     { "CLIENT GETNAME",
     "-",
     "Get the current connection name",
     9,
     "2.6.9" },
+    { "CLIENT GETREDIR",
+    "-",
+    "Get tracking notifications redirection client ID if any",
+    9,
+    "6.0.0" },
     { "CLIENT ID",
     "-",
     "Returns the client ID for the current connection",
@@ -109,7 +169,7 @@ struct commandHelp {
     9,
     "2.4.0" },
     { "CLIENT LIST",
-    "-",
+    "[TYPE normal|master|replica|pubsub]",
     "Get the list of client connections",
     9,
     "2.4.0" },
@@ -128,6 +188,11 @@ struct commandHelp {
     "Set the current connection name",
     9,
     "2.6.9" },
+    { "CLIENT TRACKING",
+    "ON|OFF [REDIRECT client-id] [PREFIX prefix] [BCAST] [OPTIN] [OPTOUT] [NOLOOP]",
+    "Enable or disable server assisted client side caching support",
+    9,
+    "6.0.0" },
     { "CLIENT UNBLOCK",
     "client-id [TIMEOUT|ERROR]",
     "Unblock a client blocked in a blocking command from a different connection",
@@ -138,6 +203,11 @@ struct commandHelp {
     "Assign new hash slots to receiving node",
     12,
     "3.0.0" },
+    { "CLUSTER BUMPEPOCH",
+    "-",
+    "Advance the cluster config epoch",
+    12,
+    "3.0.0" },
     { "CLUSTER COUNT-FAILURE-REPORTS",
     "node-id",
     "Return the number of failure reports active for a given node",
@@ -158,6 +228,11 @@ struct commandHelp {
     "Forces a replica to perform a manual failover of its master.",
     12,
     "3.0.0" },
+    { "CLUSTER FLUSHSLOTS",
+    "-",
+    "Delete a node's own slots information",
+    12,
+    "3.0.0" },
     { "CLUSTER FORGET",
     "node-id",
     "Remove a node from the nodes table",
@@ -183,6 +258,11 @@ struct commandHelp {
     "Force a node cluster to handshake with another node",
     12,
     "3.0.0" },
+    { "CLUSTER MYID",
+    "-",
+    "Return the node id",
+    12,
+    "3.0.0" },
     { "CLUSTER NODES",
     "-",
     "Get Cluster config for the node",
@@ -359,7 +439,7 @@ struct commandHelp {
     13,
     "3.2.0" },
     { "GEODIST",
-    "key member1 member2 [unit]",
+    "key member1 member2 [m|km|ft|mi]",
     "Returns the distance between two members of a geospatial index",
     13,
     "3.2.0" },
@@ -459,7 +539,7 @@ struct commandHelp {
     5,
     "2.8.0" },
     { "HSET",
-    "key field value",
+    "key field value [field value ...]",
     "Set the string value of a hash field",
     5,
     "2.0.0" },
@@ -508,13 +588,43 @@ struct commandHelp {
     "Get the UNIX time stamp of the last successful save to disk",
     9,
     "1.0.0" },
+    { "LATENCY DOCTOR",
+    "-",
+    "Return a human readable latency analysis report.",
+    9,
+    "2.8.13" },
+    { "LATENCY GRAPH",
+    "event",
+    "Return a latency graph for the event.",
+    9,
+    "2.8.13" },
+    { "LATENCY HELP",
+    "-",
+    "Show helpful text about the different subcommands.",
+    9,
+    "2.8.13" },
+    { "LATENCY HISTORY",
+    "event",
+    "Return timestamp-latency samples for the event.",
+    9,
+    "2.8.13" },
+    { "LATENCY LATEST",
+    "-",
+    "Return the latest latency samples for all events.",
+    9,
+    "2.8.13" },
+    { "LATENCY RESET",
+    "[event]",
+    "Reset latency data for one or more events.",
+    9,
+    "2.8.13" },
     { "LINDEX",
     "key index",
     "Get an element from a list by its index",
     2,
     "1.0.0" },
     { "LINSERT",
-    "key BEFORE|AFTER pivot value",
+    "key BEFORE|AFTER pivot element",
     "Insert an element before or after another element in a list",
     2,
     "2.2.0" },
@@ -523,19 +633,24 @@ struct commandHelp {
     "Get the length of a list",
     2,
     "1.0.0" },
+    { "LOLWUT",
+    "[VERSION version]",
+    "Display some computer art and the Redis version",
+    9,
+    "5.0.0" },
     { "LPOP",
     "key",
     "Remove and get the first element in a list",
     2,
     "1.0.0" },
     { "LPUSH",
-    "key value [value ...]",
-    "Prepend one or multiple values to a list",
+    "key element [element ...]",
+    "Prepend one or multiple elements to a list",
     2,
     "1.0.0" },
     { "LPUSHX",
-    "key value",
-    "Prepend a value to a list, only if the list exists",
+    "key element [element ...]",
+    "Prepend an element to a list, only if the list exists",
     2,
     "2.2.0" },
     { "LRANGE",
@@ -544,12 +659,12 @@ struct commandHelp {
     2,
     "1.0.0" },
     { "LREM",
-    "key count value",
+    "key count element",
     "Remove elements from a list",
     2,
     "1.0.0" },
     { "LSET",
-    "key index value",
+    "key index element",
     "Set the value of an element in a list by its index",
     2,
     "1.0.0" },
@@ -594,10 +709,25 @@ struct commandHelp {
     1,
     "1.0.0" },
     { "MIGRATE",
-    "host port key|"" destination-db timeout [COPY] [REPLACE] [KEYS key]",
+    "host port key|"" destination-db timeout [COPY] [REPLACE] [AUTH password] [KEYS key]",
     "Atomically transfer a key from a Redis instance to another one.",
     0,
     "2.6.0" },
+    { "MODULE LIST",
+    "-",
+    "List all modules loaded by the server",
+    9,
+    "4.0.0" },
+    { "MODULE LOAD",
+    "path [arg]",
+    "Load a module",
+    9,
+    "4.0.0" },
+    { "MODULE UNLOAD",
+    "name",
+    "Unload a module",
+    9,
+    "4.0.0" },
     { "MONITOR",
     "-",
     "Listen for all requests received by the server in real time",
@@ -673,6 +803,11 @@ struct commandHelp {
     "Listen for messages published to channels matching the given patterns",
     6,
     "2.0.0" },
+    { "PSYNC",
+    "replicationid offset",
+    "Internal command used for replication",
+    9,
+    "2.8.0" },
     { "PTTL",
     "key",
     "Get the time to live for a key in milliseconds",
@@ -729,7 +864,7 @@ struct commandHelp {
     9,
     "5.0.0" },
     { "RESTORE",
-    "key ttl serialized-value [REPLACE]",
+    "key ttl serialized-value [REPLACE] [ABSTTL] [IDLETIME seconds] [FREQ frequency]",
     "Create a key using the provided serialized value, previously obtained using DUMP.",
     0,
     "2.6.0" },
@@ -749,13 +884,13 @@ struct commandHelp {
     2,
     "1.2.0" },
     { "RPUSH",
-    "key value [value ...]",
-    "Append one or multiple values to a list",
+    "key element [element ...]",
+    "Append one or multiple elements to a list",
     2,
     "1.0.0" },
     { "RPUSHX",
-    "key value",
-    "Append a value to a list, only if the list exists",
+    "key element [element ...]",
+    "Append an element to a list, only if the list exists",
     2,
     "2.2.0" },
     { "SADD",
@@ -769,7 +904,7 @@ struct commandHelp {
     9,
     "1.0.0" },
     { "SCAN",
-    "cursor [MATCH pattern] [COUNT count]",
+    "cursor [MATCH pattern] [COUNT count] [TYPE type]",
     "Incrementally iterate the keys space",
     0,
     "2.8.0" },
@@ -819,7 +954,7 @@ struct commandHelp {
     8,
     "1.0.0" },
     { "SET",
-    "key value [expiration EX seconds|PX milliseconds] [NX|XX]",
+    "key value [EX seconds|PX milliseconds] [NX|XX] [KEEPTTL]",
     "Set the string value of a key",
     1,
     "1.0.0" },
@@ -908,6 +1043,11 @@ struct commandHelp {
     "Incrementally iterate Set elements",
     3,
     "2.8.0" },
+    { "STRALGO",
+    "LCS algo-specific-argument [algo-specific-argument ...]",
+    "Run algorithms (currently LCS) against strings",
+    1,
+    "6.0.0" },
     { "STRLEN",
     "key",
     "Get the length of the value stored in a key",
@@ -929,7 +1069,7 @@ struct commandHelp {
     3,
     "1.0.0" },
     { "SWAPDB",
-    "index index",
+    "index1 index2",
     "Swaps two Redis databases",
     8,
     "4.0.0" },
@@ -989,7 +1129,7 @@ struct commandHelp {
     14,
     "5.0.0" },
     { "XADD",
-    "key ID field string [field string ...]",
+    "key ID field value [field value ...]",
     "Appends a new entry to a stream",
     14,
     "5.0.0" },
@@ -1004,7 +1144,7 @@ struct commandHelp {
     14,
     "5.0.0" },
     { "XGROUP",
-    "[CREATE key groupname id-or-$] [SETID key id-or-$] [DESTROY key groupname] [DELCONSUMER key groupname consumername]",
+    "[CREATE key groupname id-or-$] [SETID key groupname id-or-$] [DESTROY key groupname] [DELCONSUMER key groupname consumername]",
     "Create, destroy, and manage consumer groups.",
     14,
     "5.0.0" },
@@ -1029,12 +1169,12 @@ struct commandHelp {
     14,
     "5.0.0" },
     { "XREAD",
-    "[COUNT count] [BLOCK milliseconds] STREAMS key [key ...] ID [ID ...]",
+    "[COUNT count] [BLOCK milliseconds] STREAMS key [key ...] id [id ...]",
     "Return never seen elements in multiple streams, with IDs greater than the ones reported by the caller for each stream. Can block.",
     14,
     "5.0.0" },
     { "XREADGROUP",
-    "GROUP group consumer [COUNT count] [BLOCK milliseconds] STREAMS key [key ...] ID [ID ...]",
+    "GROUP group consumer [COUNT count] [BLOCK milliseconds] [NOACK] STREAMS key [key ...] ID [ID ...]",
     "Return new entries from a stream using a consumer group, or access the history of the pending entries for a given consumer. Can block.",
     14,
     "5.0.0" },

From 3fcffe7d0496a0cb0cceb7b9933947ffdc9692eb Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Thu, 30 Apr 2020 13:42:25 +0200
Subject: [PATCH 118/186] redis-cli: fix hints with subcommands.

---
 src/redis-cli.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/redis-cli.c b/src/redis-cli.c
index ec0f58f5f..880f3d70a 100644
--- a/src/redis-cli.c
+++ b/src/redis-cli.c
@@ -699,7 +699,8 @@ static char *hintsCallback(const char *buf, int *color, int *bold) {
     for (i = 0; i < helpEntriesLen; i++) {
         if (!(helpEntries[i].type & CLI_HELP_COMMAND)) continue;
 
-        if (strcasecmp(argv[0],helpEntries[i].full) == 0)
+        if (strcasecmp(argv[0],helpEntries[i].full) == 0 ||
+            strcasecmp(buf,helpEntries[i].full) == 0)
         {
             *color = 90;
             *bold = 0;

From 1f9b82bd568cf8d401a8f2b93b9e66fe4bd4abfc Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Thu, 30 Apr 2020 13:43:05 +0200
Subject: [PATCH 119/186] Update help.h again before Redis 6 GA.

---
 src/help.h | 29 +++++++++++++++++------------
 1 file changed, 17 insertions(+), 12 deletions(-)

diff --git a/src/help.h b/src/help.h
index 6aa875541..6d3eb33ed 100644
--- a/src/help.h
+++ b/src/help.h
@@ -146,57 +146,57 @@ struct commandHelp {
     { "CLIENT CACHING",
     "YES|NO",
     "Instruct the server about tracking or not keys in the next request",
-    9,
+    8,
     "6.0.0" },
     { "CLIENT GETNAME",
     "-",
     "Get the current connection name",
-    9,
+    8,
     "2.6.9" },
     { "CLIENT GETREDIR",
     "-",
     "Get tracking notifications redirection client ID if any",
-    9,
+    8,
     "6.0.0" },
     { "CLIENT ID",
     "-",
     "Returns the client ID for the current connection",
-    9,
+    8,
     "5.0.0" },
     { "CLIENT KILL",
     "[ip:port] [ID client-id] [TYPE normal|master|slave|pubsub] [ADDR ip:port] [SKIPME yes/no]",
     "Kill the connection of a client",
-    9,
+    8,
     "2.4.0" },
     { "CLIENT LIST",
     "[TYPE normal|master|replica|pubsub]",
     "Get the list of client connections",
-    9,
+    8,
     "2.4.0" },
     { "CLIENT PAUSE",
     "timeout",
     "Stop processing commands from clients for some time",
-    9,
+    8,
     "2.9.50" },
     { "CLIENT REPLY",
     "ON|OFF|SKIP",
     "Instruct the server whether to reply to commands",
-    9,
+    8,
     "3.2" },
     { "CLIENT SETNAME",
     "connection-name",
     "Set the current connection name",
-    9,
+    8,
     "2.6.9" },
     { "CLIENT TRACKING",
     "ON|OFF [REDIRECT client-id] [PREFIX prefix] [BCAST] [OPTIN] [OPTOUT] [NOLOOP]",
     "Enable or disable server assisted client side caching support",
-    9,
+    8,
     "6.0.0" },
     { "CLIENT UNBLOCK",
     "client-id [TIMEOUT|ERROR]",
     "Unblock a client blocked in a blocking command from a different connection",
-    9,
+    8,
     "5.0.0" },
     { "CLUSTER ADDSLOTS",
     "slot [slot ...]",
@@ -488,6 +488,11 @@ struct commandHelp {
     "Delete one or more hash fields",
     5,
     "2.0.0" },
+    { "HELLO",
+    "protover [AUTH username password] [SETNAME clientname]",
+    "switch Redis protocol",
+    8,
+    "6.0.0" },
     { "HEXISTS",
     "key field",
     "Determine if a hash field exists",
@@ -1071,7 +1076,7 @@ struct commandHelp {
     { "SWAPDB",
     "index1 index2",
     "Swaps two Redis databases",
-    8,
+    9,
     "4.0.0" },
     { "SYNC",
     "-",

From 17dfd7cabbf7954f92b7a1243d4bb27fee5d4500 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Thu, 30 Apr 2020 15:04:41 +0200
Subject: [PATCH 120/186] Redis 6.0.0 GA.

---
 00-RELEASENOTES | 263 ++++++++++++++++++++++++++++++++++++++++++++++++
 src/version.h   |   2 +-
 2 files changed, 264 insertions(+), 1 deletion(-)

diff --git a/00-RELEASENOTES b/00-RELEASENOTES
index 223e6e66c..2b67732a1 100644
--- a/00-RELEASENOTES
+++ b/00-RELEASENOTES
@@ -11,6 +11,269 @@ CRITICAL: There is a critical bug affecting MOST USERS. Upgrade ASAP.
 SECURITY: There are security fixes in the release.
 --------------------------------------------------------------------------------
 
+================================================================================
+Redis 6.0.0 GA  Released Thu Apr 30 14:55:02 CEST 2020
+================================================================================
+
+Upgrade urgency CRITICAL: many bugs fixed compared to the last release
+                          candidate. Better to upgrade if you see things
+                          affecting your environment in the changelog.
+
+Hi all, finally we have Redis 6.0.0 GA! Enjoy this new Redis release.
+Most of the documentation was updated today so that you can likely
+find what you are looking for about the new features at redis.io.
+This is the list of what changed compared to the previoius release candidate:
+
+* XCLAIM AOF/replicas propagation fixed.
+* Client side caching: new NOLOOP option to avoid getting notified about
+  changes performed by ourselves.
+* ACL GENPASS now uses HMAC-SHA256 and have an optional "bits" argument.
+  It means you can use it as a general purpose "secure random strings"
+  primitive!
+* Cluster "SLOTS" subcommand memory optimization.
+* The LCS command is now a subcommand of STRALGO.
+* Meaningful offset for replicas as well. More successful partial
+  resynchronizations.
+* Optimize memory usage of deferred replies.
+* Faster CRC64 algorithm for faster RDB loading.
+* XINFO STREAM FULL, a new subcommand to get the whole stream state.
+* CLIENT KILL USER <username>.
+* MIGRATE AUTH2 option, for ACL style authentication support.
+* Other random bugfixes.
+
+Enjoy Redis 6! :-)
+Goodbye antirez
+
+List of commits in this release:
+
+antirez in commit 1f9b82bd5:
+ Update help.h again before Redis 6 GA.
+ 1 file changed, 17 insertions(+), 12 deletions(-)
+
+antirez in commit 3fcffe7d0:
+ redis-cli: fix hints with subcommands.
+ 1 file changed, 2 insertions(+), 1 deletion(-)
+
+antirez in commit 455d8a05c:
+ redis-cli command help updated.
+ 1 file changed, 165 insertions(+), 25 deletions(-)
+
+zhaozhao.zz in commit 70287bbc9:
+ lazyfree & eviction: record latency generated by lazyfree eviction
+ 1 file changed, 18 insertions(+), 13 deletions(-)
+
+antirez in commit 7be21139a:
+ MIGRATE AUTH2 for ACL support.
+ 1 file changed, 19 insertions(+), 5 deletions(-)
+
+antirez in commit e1ee1a49d:
+ CLIENT KILL USER <username>.
+ 1 file changed, 11 insertions(+)
+
+antirez in commit d56f058c0:
+ Fix tracking table max keys option in redis.conf.
+ 1 file changed, 12 insertions(+), 9 deletions(-)
+
+antirez in commit 96dd5fc93:
+ redis-cli: safer cluster fix with unreachalbe masters.
+ 1 file changed, 26 insertions(+), 1 deletion(-)
+
+antirez in commit 5b59d9c5d:
+ redis-cli: simplify cluster nodes coverage display.
+ 1 file changed, 10 insertions(+), 17 deletions(-)
+
+antirez in commit c163d4add:
+ redis-cli: try to make clusterManagerFixOpenSlot() more readable.
+ 1 file changed, 25 insertions(+), 6 deletions(-)
+
+Guy Benoish in commit aab74b715:
+ XINFO STREAM FULL should have a default COUNT of 10
+ 1 file changed, 8 insertions(+), 4 deletions(-)
+
+antirez in commit 606134f9d:
+ Comment clearly why we moved some code in #6623.
+ 1 file changed, 4 insertions(+), 1 deletion(-)
+
+srzhao in commit ee627bb66:
+ fix pipelined WAIT performance issue.
+ 1 file changed, 13 insertions(+), 13 deletions(-)
+
+antirez in commit 47b8a7f9b:
+ Fix create-cluster BIN_PATH.
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+Guy Benoish in commit 6c0bc608a:
+ Extend XINFO STREAM output
+ 2 files changed, 226 insertions(+), 34 deletions(-)
+
+hwware in commit 5bfc18950:
+ Fix not used marco in cluster.c
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+Itamar Haber in commit 56d628f85:
+ Update create-cluster
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+Itamar Haber in commit cac9d7cf7:
+ Adds `BIN_PATH` to create-cluster
+ 1 file changed, 8 insertions(+), 6 deletions(-)
+
+Oran Agra in commit b712fba17:
+ hickup, re-fix dictEncObjKeyCompare
+ 1 file changed, 4 insertions(+), 4 deletions(-)
+
+Oran Agra in commit ea63aea72:
+ fix loading race in psync2 tests
+ 3 files changed, 15 insertions(+), 1 deletion(-)
+
+antirez in commit 64e588bfa:
+ Rework comment in dictEncObjKeyCompare().
+ 1 file changed, 8 insertions(+), 9 deletions(-)
+
+Oran Agra in commit 0d1e8c93b:
+ allow dictFind using static robj
+ 1 file changed, 9 insertions(+), 4 deletions(-)
+
+Madelyn Olson in commit a1bed447b:
+ Added crcspeed library
+ 2 files changed, 341 insertions(+)
+
+Madelyn Olson in commit a75fa3aad:
+ Made crc64 test consistent
+ 1 file changed, 3 insertions(+), 2 deletions(-)
+
+Madelyn Olson in commit 52c75e9db:
+ Implemented CRC64 based on slice by 4
+ 5 files changed, 124 insertions(+), 157 deletions(-)
+
+Oran Agra in commit 8110ba888:
+ optimize memory usage of deferred replies
+ 1 file changed, 31 insertions(+)
+
+Oran Agra in commit e4d2bb62b:
+ Keep track of meaningful replication offset in replicas too
+ 5 files changed, 212 insertions(+), 92 deletions(-)
+
+antirez in commit fea9788cc:
+ Fix STRALGO command flags.
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+Dave-in-lafayette in commit 2144047e1:
+ fix for unintended crash during panic response
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+Guy Benoish in commit 43329c9b6:
+ Add the stream tag to XSETID tests
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+Dave-in-lafayette in commit 1e17d3de7:
+ fix for crash during panic before all threads are up
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+antirez in commit 3722f89f4:
+ LCS -> STRALGO LCS.
+ 4 files changed, 28 insertions(+), 15 deletions(-)
+
+antirez in commit 373ae6061:
+ Also use propagate() in streamPropagateGroupID().
+ 1 file changed, 11 insertions(+), 1 deletion(-)
+
+yanhui13 in commit f03f1fad6:
+ add tcl test for cluster slots
+ 1 file changed, 44 insertions(+)
+
+yanhui13 in commit 374ffdf1c:
+ optimize the output of cluster slots
+ 1 file changed, 7 insertions(+), 4 deletions(-)
+
+antirez in commit 4db38d2ef:
+ Minor aesthetic changes to #7135.
+ 1 file changed, 5 insertions(+), 7 deletions(-)
+
+Valentino Geron in commit f0a261448:
+ XREADGROUP with NOACK should propagate only one XGROUP SETID command
+ 1 file changed, 13 insertions(+), 7 deletions(-)
+
+antirez in commit fbdef6a9b:
+ ACL: re-enable command execution of disabled users.
+ 1 file changed, 4 deletions(-)
+
+antirez in commit 05a41da75:
+ getRandomBytes(): use HMAC-SHA256.
+ 1 file changed, 30 insertions(+), 10 deletions(-)
+
+antirez in commit 345c3768d:
+ ACL GENPASS: take number of bits as argument.
+ 1 file changed, 21 insertions(+), 6 deletions(-)
+
+antirez in commit 639c8a1d9:
+ ACL GENPASS: emit 256 bits instead of 128.
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+antirez in commit 321acea03:
+ ACL: deny commands execution of disabled users.
+ 1 file changed, 4 insertions(+)
+
+Theo Buehler in commit b0920e6e8:
+ TLS: Fix build with SSL_OP_NO_CLIENT_RENEGOTIATION
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+Yossi Gottlieb in commit 149b658b5:
+ TLS: Fix build on older verisons of OpenSSL.
+ 1 file changed, 2 insertions(+)
+
+antirez in commit 06917e581:
+ Tracking: test expired keys notifications.
+ 1 file changed, 13 insertions(+)
+
+antirez in commit e434b2ce4:
+ Tracking: NOLOOP tests.
+ 1 file changed, 32 insertions(+)
+
+antirez in commit f3a172887:
+ Tracking: signal key as modified when evicting.
+ 1 file changed, 1 insertion(+)
+
+antirez in commit e63bb7ec8:
+ Tracking: NOLOOP further implementation and fixes.
+ 2 files changed, 21 insertions(+), 6 deletions(-)
+
+antirez in commit 6791ff052:
+ Tracking: NOLOOP internals implementation.
+ 17 files changed, 174 insertions(+), 112 deletions(-)
+
+antirez in commit 725b8cc68:
+ Implement redis_set_thread_title for MacOS.
+ 1 file changed, 6 insertions(+)
+
+zhenwei pi in commit 3575b8706:
+ Threaded IO: set thread name for redis-server
+ 3 files changed, 28 insertions(+)
+
+antirez in commit a76c67578:
+ Sentinel: small refactoring of sentinelCollectTerminatedScripts().
+ 1 file changed, 1 insertion(+), 2 deletions(-)
+
+omg-by in commit 3a27064c4:
+ fix(sentinel): sentinel.running_scripts will always increase more times and not reset
+ 1 file changed, 1 insertion(+)
+
+antirez in commit 5c4c73e2c:
+ A few comments and name changes for #7103.
+ 1 file changed, 13 insertions(+), 4 deletions(-)
+
+Oran Agra in commit 6148f9493:
+ testsuite run the defrag latency test solo
+ 3 files changed, 42 insertions(+), 2 deletions(-)
+
+Jamie Scott in commit 51d3012d4:
+ Adding acllog-max-len to Redis.conf
+ 1 file changed, 9 insertions(+)
+
+antirez in commit c39f16c42:
+ Fix XCLAIM propagation in AOF/replicas for blocking XREADGROUP.
+ 2 files changed, 8 insertions(+), 3 deletions(-)
+
 ================================================================================
 Redis 6.0-rc4     Released Thu Apr 16 16:10:35 CEST 2020
 ================================================================================
diff --git a/src/version.h b/src/version.h
index c240a174e..8236702db 100644
--- a/src/version.h
+++ b/src/version.h
@@ -1 +1 @@
-#define REDIS_VERSION "5.9.104"
+#define REDIS_VERSION "6.0.0"

From 8fe25edc7560eac183df31e2831a60777406fff9 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Fri, 1 May 2020 18:45:51 +0200
Subject: [PATCH 121/186] Save a call to stopThreadedIOIfNeeded() for the base
 case.

Probably no performance changes, but the code should be trivial to
read as in "No threading? Use the normal function and return".
---
 src/networking.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/networking.c b/src/networking.c
index 767206ab9..d4c0306c3 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -2997,9 +2997,9 @@ int handleClientsWithPendingWritesUsingThreads(void) {
     int processed = listLength(server.clients_pending_write);
     if (processed == 0) return 0; /* Return ASAP if there are no clients. */
 
-    /* If we have just a few clients to serve, don't use I/O threads, but the
-     * boring synchronous code. */
-    if (stopThreadedIOIfNeeded()) {
+    /* If I/O threads are disabled or we have few clients to serve, don't
+     * use I/O threads, but thejboring synchronous code. */
+    if (server.io_threads_num == 1 || stopThreadedIOIfNeeded()) {
         return handleClientsWithPendingWrites();
     }
 

From 7c0fe727145e5927d4ebeaff0bdcdfef061ec999 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Fri, 1 May 2020 23:57:11 +0200
Subject: [PATCH 122/186] Revert "optimize memory usage of deferred replies"

This reverts commit fb732f7a944a4d4c90bb7375cb6030e88211f5aa.
---
 src/networking.c | 31 -------------------------------
 1 file changed, 31 deletions(-)

diff --git a/src/networking.c b/src/networking.c
index d4c0306c3..c4a277e0a 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -436,36 +436,6 @@ void addReplyStatusFormat(client *c, const char *fmt, ...) {
     sdsfree(s);
 }
 
-/* Sometimes we are forced to create a new reply node, and we can't append to
- * the previous one, when that happens, we wanna try to trim the unused space
- * at the end of the last reply node which we won't use anymore. */
-void trimReplyUnusedTailSpace(client *c) {
-    listNode *ln = listLast(c->reply);
-    clientReplyBlock *tail = ln? listNodeValue(ln): NULL;
-
-    /* Note that 'tail' may be NULL even if we have a tail node, becuase when
-     * addDeferredMultiBulkLength() is used */
-    if (!tail) return;
-
-    /* We only try to trim the space is relatively high (more than a 1/4 of the
-     * allocation), otherwise there's a high chance realloc will NOP.
-     * Also, to avoid large memmove which happens as part of realloc, we only do
-     * that if the used part is small.  */
-    if (tail->size - tail->used > tail->size / 4 &&
-        tail->used < PROTO_REPLY_CHUNK_BYTES)
-    {
-        size_t old_size = tail->size;
-        tail = zrealloc(tail, tail->used + sizeof(clientReplyBlock));
-        /* If realloc was a NOP, we got the same value which has internal frag */
-        if (tail == listNodeValue(ln)) return;
-        /* take over the allocation's internal fragmentation (at least for
-         * memory usage tracking) */
-        tail->size = zmalloc_usable(tail) - sizeof(clientReplyBlock);
-        c->reply_bytes += tail->size - old_size;
-        listNodeValue(ln) = tail;
-    }
-}
-
 /* Adds an empty object to the reply list that will contain the multi bulk
  * length, which is not known when this function is called. */
 void *addReplyDeferredLen(client *c) {
@@ -473,7 +443,6 @@ void *addReplyDeferredLen(client *c) {
      * ready to be sent, since we are sure that before returning to the
      * event loop setDeferredAggregateLen() will be called. */
     if (prepareClientToWrite(c) != C_OK) return NULL;
-    trimReplyUnusedTailSpace(c);
     listAddNodeTail(c->reply,NULL); /* NULL is our placeholder. */
     return listLast(c->reply);
 }

From db73d0998fc031b6dceb886cad2128ade0771118 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Sat, 2 May 2020 00:02:18 +0200
Subject: [PATCH 123/186] Cast printf() argument to the format specifier.

We could use uint64_t specific macros, but after all it's simpler to
just use an obvious equivalent type plus casting: this will be a no op
and is simpler than fixed size types printf macros.
---
 src/rdb.c | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/rdb.c b/src/rdb.c
index 9f6bf13f1..4fc385848 100644
--- a/src/rdb.c
+++ b/src/rdb.c
@@ -2292,7 +2292,9 @@ int rdbLoadRio(rio *rdb, int rdbflags, rdbSaveInfo *rsi) {
                 serverLog(LL_WARNING,"RDB file was saved with checksum disabled: no check performed.");
             } else if (cksum != expected) {
                 serverLog(LL_WARNING,"Wrong RDB checksum expected: (%llx) but "
-                    "got (%llx). Aborting now.",expected,cksum);
+                    "got (%llx). Aborting now.",
+                        (unsigned long long)expected,
+                        (unsigned long long)cksum);
                 rdbExitReportCorruptRDB("RDB CRC error");
             }
         }

From 47ff13637e1254e5a6ac962afa3e963d2f284783 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Sat, 2 May 2020 00:10:20 +0200
Subject: [PATCH 124/186] Redis 6.0.1.

---
 00-RELEASENOTES | 28 ++++++++++++++++++++++++++++
 src/version.h   |  2 +-
 2 files changed, 29 insertions(+), 1 deletion(-)

diff --git a/00-RELEASENOTES b/00-RELEASENOTES
index 2b67732a1..1d5486ae8 100644
--- a/00-RELEASENOTES
+++ b/00-RELEASENOTES
@@ -11,6 +11,34 @@ CRITICAL: There is a critical bug affecting MOST USERS. Upgrade ASAP.
 SECURITY: There are security fixes in the release.
 --------------------------------------------------------------------------------
 
+================================================================================
+Redis 6.0.1     Released Sat May 02 00:06:07 CEST 2020
+================================================================================
+
+Upgrade urgency HIGH: This release fixes a crash when builiding against
+                      Libc malloc.
+
+Here we revert 8110ba888, an optimization that causes a crash due to a
+bug in the code. It does not happen with the default allocator because of
+differences between Jemalloc and libc malloc, so this escaped all our
+testing but was reported by a user. We'll add back the original optimization
+that was reverted here later, after checking what happens: it is not a
+critical optimization.
+
+The other commits are minor stuff:
+
+antirez in commit db73d0998:
+ Cast printf() argument to the format specifier.
+ 1 file changed, 3 insertions(+), 1 deletion(-)
+
+antirez in commit 7c0fe7271:
+ Revert "optimize memory usage of deferred replies"
+ 1 file changed, 31 deletions(-)
+
+antirez in commit 8fe25edc7:
+ Save a call to stopThreadedIOIfNeeded() for the base case.
+ 1 file changed, 3 insertions(+), 3 deletions(-)
+
 ================================================================================
 Redis 6.0.0 GA  Released Thu Apr 30 14:55:02 CEST 2020
 ================================================================================
diff --git a/src/version.h b/src/version.h
index 8236702db..91d3566d3 100644
--- a/src/version.h
+++ b/src/version.h
@@ -1 +1 @@
-#define REDIS_VERSION "6.0.0"
+#define REDIS_VERSION "6.0.1"

From d6436eb7cf690afee69a4a3b9b61bf3241eed645 Mon Sep 17 00:00:00 2001
From: zhenwei pi <pizhenwei@bytedance.com>
Date: Sat, 2 May 2020 20:05:39 +0800
Subject: [PATCH 125/186] Support setcpuaffinity on linux/bsd

Currently, there are several types of threads/child processes of a
redis server. Sometimes we need deeply optimise the performance of
redis, so we would like to isolate threads/processes.

There were some discussion about cpu affinity cases in the issue:
https://github.com/antirez/redis/issues/2863

So implement cpu affinity setting by redis.conf in this patch, then
we can config server_cpulist/bio_cpulist/aof_rewrite_cpulist/
bgsave_cpulist by cpu list.

Examples of cpulist in redis.conf:
server_cpulist 0-7:2      means cpu affinity 0,2,4,6
bio_cpulist 1,3           means cpu affinity 1,3
aof_rewrite_cpulist 8-11  means cpu affinity 8,9,10,11
bgsave_cpulist 1,10-11    means cpu affinity 1,10,11

Test on linux/freebsd, both work fine.

Signed-off-by: zhenwei pi <pizhenwei@bytedance.com>
---
 redis.conf                   |  15 ++++
 src/Makefile                 |   2 +-
 src/aof.c                    |   1 +
 src/bio.c                    |   2 +
 src/config.c                 |   4 ++
 src/config.h                 |   6 ++
 src/networking.c             |   1 +
 src/rdb.c                    |   2 +
 src/server.c                 |   9 +++
 src/server.h                 |   6 ++
 src/setcpuaffinity.c         | 129 +++++++++++++++++++++++++++++++++++
 tests/unit/introspection.tcl |   4 ++
 12 files changed, 180 insertions(+), 1 deletion(-)
 create mode 100644 src/setcpuaffinity.c

diff --git a/redis.conf b/redis.conf
index 6cec636bc..9c6171d4c 100644
--- a/redis.conf
+++ b/redis.conf
@@ -1781,3 +1781,18 @@ rdb-save-incremental-fsync yes
 # Maximum number of set/hash/zset/list fields that will be processed from
 # the main dictionary scan
 # active-defrag-max-scan-fields 1000
+
+# Redis server/IO threads, bio threads, aof rewrite child process, and bgsave
+# child process cpu affinity list config. syntax of cpu list looks like taskset
+# command. serveral examples:
+# set redis server/io threads to cpu affinity 0,2,4,6
+# server_cpulist 0-7:2
+#
+# set bio threads to cpu affinity 1,3
+# bio_cpulist 1,3
+#
+# set aof rewrite child process to cpu affinity 8,9,10,11
+# aof_rewrite_cpulist 8-11
+#
+# set bgsave child process to cpu affinity 1,10,11
+# bgsave_cpulist 1,10-11
diff --git a/src/Makefile b/src/Makefile
index f9922afce..55f862cfc 100644
--- a/src/Makefile
+++ b/src/Makefile
@@ -206,7 +206,7 @@ endif
 
 REDIS_SERVER_NAME=redis-server
 REDIS_SENTINEL_NAME=redis-sentinel
-REDIS_SERVER_OBJ=adlist.o quicklist.o ae.o anet.o dict.o server.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o cluster.o crc16.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o memtest.o crcspeed.o crc64.o bitops.o sentinel.o notify.o setproctitle.o blocked.o hyperloglog.o latency.o sparkline.o redis-check-rdb.o redis-check-aof.o geo.o lazyfree.o module.o evict.o expire.o geohash.o geohash_helper.o childinfo.o defrag.o siphash.o rax.o t_stream.o listpack.o localtime.o lolwut.o lolwut5.o lolwut6.o acl.o gopher.o tracking.o connection.o tls.o sha256.o timeout.o
+REDIS_SERVER_OBJ=adlist.o quicklist.o ae.o anet.o dict.o server.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o cluster.o crc16.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o memtest.o crcspeed.o crc64.o bitops.o sentinel.o notify.o setproctitle.o blocked.o hyperloglog.o latency.o sparkline.o redis-check-rdb.o redis-check-aof.o geo.o lazyfree.o module.o evict.o expire.o geohash.o geohash_helper.o childinfo.o defrag.o siphash.o rax.o t_stream.o listpack.o localtime.o lolwut.o lolwut5.o lolwut6.o acl.o gopher.o tracking.o connection.o tls.o sha256.o timeout.o setcpuaffinity.o
 REDIS_CLI_NAME=redis-cli
 REDIS_CLI_OBJ=anet.o adlist.o dict.o redis-cli.o zmalloc.o release.o ae.o crcspeed.o crc64.o siphash.o crc16.o
 REDIS_BENCHMARK_NAME=redis-benchmark
diff --git a/src/aof.c b/src/aof.c
index 301a40848..02409abe6 100644
--- a/src/aof.c
+++ b/src/aof.c
@@ -1596,6 +1596,7 @@ int rewriteAppendOnlyFileBackground(void) {
 
         /* Child */
         redisSetProcTitle("redis-aof-rewrite");
+        redisSetCpuAffinity(server.aof_rewrite_cpulist);
         snprintf(tmpfile,256,"temp-rewriteaof-bg-%d.aof", (int) getpid());
         if (rewriteAppendOnlyFile(tmpfile) == C_OK) {
             sendChildCOWInfo(CHILD_INFO_TYPE_AOF, "AOF rewrite");
diff --git a/src/bio.c b/src/bio.c
index 85f681185..69c62fc6f 100644
--- a/src/bio.c
+++ b/src/bio.c
@@ -166,6 +166,8 @@ void *bioProcessBackgroundJobs(void *arg) {
         break;
     }
 
+    redisSetCpuAffinity(server.bio_cpulist);
+
     /* Make the thread killable at any time, so that bioKillThreads()
      * can work reliably. */
     pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL);
diff --git a/src/config.c b/src/config.c
index e0cbcc281..64854592c 100644
--- a/src/config.c
+++ b/src/config.c
@@ -2133,6 +2133,10 @@ standardConfig configs[] = {
     createStringConfig("syslog-ident", NULL, IMMUTABLE_CONFIG, ALLOW_EMPTY_STRING, server.syslog_ident, "redis", NULL, NULL),
     createStringConfig("dbfilename", NULL, MODIFIABLE_CONFIG, ALLOW_EMPTY_STRING, server.rdb_filename, "dump.rdb", isValidDBfilename, NULL),
     createStringConfig("appendfilename", NULL, IMMUTABLE_CONFIG, ALLOW_EMPTY_STRING, server.aof_filename, "appendonly.aof", isValidAOFfilename, NULL),
+    createStringConfig("server_cpulist", NULL, IMMUTABLE_CONFIG, EMPTY_STRING_IS_NULL, server.server_cpulist, NULL, NULL, NULL),
+    createStringConfig("bio_cpulist", NULL, IMMUTABLE_CONFIG, EMPTY_STRING_IS_NULL, server.bio_cpulist, NULL, NULL, NULL),
+    createStringConfig("aof_rewrite_cpulist", NULL, IMMUTABLE_CONFIG, EMPTY_STRING_IS_NULL, server.aof_rewrite_cpulist, NULL, NULL, NULL),
+    createStringConfig("bgsave_cpulist", NULL, IMMUTABLE_CONFIG, EMPTY_STRING_IS_NULL, server.bgsave_cpulist, NULL, NULL, NULL),
 
     /* Enum Configs */
     createEnumConfig("supervised", NULL, IMMUTABLE_CONFIG, supervised_mode_enum, server.supervised_mode, SUPERVISED_NONE, NULL, NULL),
diff --git a/src/config.h b/src/config.h
index 40dc683ce..a322ce354 100644
--- a/src/config.h
+++ b/src/config.h
@@ -244,4 +244,10 @@ int pthread_setname_np(const char *name);
 #endif
 #endif
 
+/* Check if we can use setcpuaffinity(). */
+#if (defined __linux || defined __NetBSD__ || defined __FreeBSD__ || defined __OpenBSD__)
+#define USE_SETCPUAFFINITY
+void setcpuaffinity(const char *cpulist);
+#endif
+
 #endif
diff --git a/src/networking.c b/src/networking.c
index d62533e3e..fd1159e06 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -2872,6 +2872,7 @@ void *IOThreadMain(void *myid) {
 
     snprintf(thdname, sizeof(thdname), "io_thd_%ld", id);
     redis_set_thread_title(thdname);
+    redisSetCpuAffinity(server.server_cpulist);
 
     while(1) {
         /* Wait for start */
diff --git a/src/rdb.c b/src/rdb.c
index c882efcb4..5cec208c5 100644
--- a/src/rdb.c
+++ b/src/rdb.c
@@ -1351,6 +1351,7 @@ int rdbSaveBackground(char *filename, rdbSaveInfo *rsi) {
 
         /* Child */
         redisSetProcTitle("redis-rdb-bgsave");
+        redisSetCpuAffinity(server.bgsave_cpulist);
         retval = rdbSave(filename,rsi);
         if (retval == C_OK) {
             sendChildCOWInfo(CHILD_INFO_TYPE_RDB, "RDB");
@@ -2488,6 +2489,7 @@ int rdbSaveToSlavesSockets(rdbSaveInfo *rsi) {
         rioInitWithFd(&rdb,server.rdb_pipe_write);
 
         redisSetProcTitle("redis-rdb-to-slaves");
+        redisSetCpuAffinity(server.bgsave_cpulist);
 
         retval = rdbSaveRioWithEOFMark(&rdb,NULL,rsi);
         if (retval == C_OK && rioFlush(&rdb) == 0)
diff --git a/src/server.c b/src/server.c
index 659604ef3..1baa044be 100644
--- a/src/server.c
+++ b/src/server.c
@@ -4850,6 +4850,14 @@ void redisSetProcTitle(char *title) {
 #endif
 }
 
+void redisSetCpuAffinity(const char *cpulist) {
+#ifdef USE_SETCPUAFFINITY
+    setcpuaffinity(cpulist);
+#else
+    UNUSED(cpulist);
+#endif
+}
+
 /*
  * Check whether systemd or upstart have been used to start redis.
  */
@@ -5118,6 +5126,7 @@ int main(int argc, char **argv) {
         serverLog(LL_WARNING,"WARNING: You specified a maxmemory value that is less than 1MB (current value is %llu bytes). Are you sure this is what you really want?", server.maxmemory);
     }
 
+    redisSetCpuAffinity(server.server_cpulist);
     aeSetBeforeSleepProc(server.el,beforeSleep);
     aeSetAfterSleepProc(server.el,afterSleep);
     aeMain(server.el);
diff --git a/src/server.h b/src/server.h
index 41d767e13..af435b148 100644
--- a/src/server.h
+++ b/src/server.h
@@ -1433,6 +1433,11 @@ struct redisServer {
     int tls_replication;
     int tls_auth_clients;
     redisTLSContextConfig tls_ctx_config;
+    /* cpu affinity */
+    char *server_cpulist; /* cpu affinity list of redis server main/io thread. */
+    char *bio_cpulist; /* cpu affinity list of bio thread. */
+    char *aof_rewrite_cpulist; /* cpu affinity list of aof rewrite process. */
+    char *bgsave_cpulist; /* cpu affinity list of bgsave process. */
 };
 
 typedef struct pubsubPattern {
@@ -1585,6 +1590,7 @@ void exitFromChild(int retcode);
 size_t redisPopcount(void *s, long count);
 void redisSetProcTitle(char *title);
 int redisCommunicateSystemd(const char *sd_notify_msg);
+void redisSetCpuAffinity(const char *cpulist);
 
 /* networking.c -- Networking and Client related operations */
 client *createClient(connection *conn);
diff --git a/src/setcpuaffinity.c b/src/setcpuaffinity.c
new file mode 100644
index 000000000..1a12795cd
--- /dev/null
+++ b/src/setcpuaffinity.c
@@ -0,0 +1,129 @@
+/* ==========================================================================
+ * setproctitle.c - Linux/BSD setcpuaffinity.
+ * --------------------------------------------------------------------------
+ * Copyright (C) 2020  zhenwei pi
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sublicense, and/or sell copies of the Software, and to permit
+ * persons to whom the Software is furnished to do so, subject to the
+ * following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included
+ * in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
+ * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+ * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+ * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
+ * USE OR OTHER DEALINGS IN THE SOFTWARE.
+ * ==========================================================================
+ */
+#ifndef _GNU_SOURCE
+#define _GNU_SOURCE
+#endif
+#include <stdlib.h>
+#include <string.h>
+#include <ctype.h>
+#ifdef __linux__
+#include <sched.h>
+#endif
+#ifdef __FreeBSD__
+#include <sys/param.h>
+#include <sys/cpuset.h>
+#endif
+#include "config.h"
+
+#ifdef USE_SETCPUAFFINITY
+static const char *next_token(const char *q,  int sep) {
+    if (q)
+        q = strchr(q, sep);
+    if (q)
+        q++;
+
+    return q;
+}
+
+static int next_num(const char *str, char **end, int *result) {
+    if (!str || *str == '\0' || !isdigit(*str))
+        return -1;
+
+    *result = strtoul(str, end, 10);
+    if (str == *end)
+        return -1;
+
+    return 0;
+}
+
+/* set current thread cpu affinity to cpu list, this function works like
+ * taskset command (actually cpulist parsing logic reference to util-linux).
+ * example of this function: "0,2,3", "0,2-3", "0-20:2". */
+void setcpuaffinity(const char *cpulist) {
+    const char *p, *q;
+    char *end = NULL;
+#ifdef __linux__
+    cpu_set_t cpuset;
+#endif
+#ifdef __FreeBSD__
+    cpuset_t cpuset;
+#endif
+
+    if (!cpulist)
+        return;
+
+    CPU_ZERO(&cpuset);
+
+    q = cpulist;
+    while (p = q, q = next_token(q, ','), p) {
+        int a, b, s;
+        const char *c1, *c2;
+
+        if (next_num(p, &end, &a) != 0)
+            return;
+
+        b = a;
+        s = 1;
+        p = end;
+
+        c1 = next_token(p, '-');
+        c2 = next_token(p, ',');
+
+        if (c1 != NULL && (c2 == NULL || c1 < c2)) {
+            if (next_num(c1, &end, &b) != 0)
+                return;
+
+            c1 = end && *end ? next_token(end, ':') : NULL;
+            if (c1 != NULL && (c2 == NULL || c1 < c2)) {
+                if (next_num(c1, &end, &s) != 0)
+                    return;
+
+                if (s == 0)
+                    return;
+            }
+        }
+
+        if ((a > b))
+            return;
+
+        while (a <= b) {
+            CPU_SET(a, &cpuset);
+            a += s;
+        }
+    }
+
+    if (end && *end)
+        return;
+
+#ifdef __linux__
+    sched_setaffinity(0, sizeof(cpuset), &cpuset);
+#endif
+#ifdef __FreeBSD__
+    cpuset_setaffinity(CPU_LEVEL_WHICH, CPU_WHICH_TID, -1, sizeof(cpuset), &cpuset);
+#endif
+}
+
+#endif /* USE_SETCPUAFFINITY */
diff --git a/tests/unit/introspection.tcl b/tests/unit/introspection.tcl
index cd905084a..b60ca0d48 100644
--- a/tests/unit/introspection.tcl
+++ b/tests/unit/introspection.tcl
@@ -94,6 +94,10 @@ start_server {tags {"introspection"}} {
             slaveof
             bind
             requirepass
+            server_cpulist
+            bio_cpulist
+            aof_rewrite_cpulist
+            bgsave_cpulist
         }
 
         set configs {}

From 4efb25d9c0193954c9b25c05350aa910e7f9758b Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Mon, 4 May 2020 11:05:48 +0200
Subject: [PATCH 126/186] Rework a bit the documentation for CPU pinning.

---
 redis.conf           | 24 +++++++++++++++++-------
 src/setcpuaffinity.c |  2 +-
 2 files changed, 18 insertions(+), 8 deletions(-)

diff --git a/redis.conf b/redis.conf
index 9c6171d4c..bed906058 100644
--- a/redis.conf
+++ b/redis.conf
@@ -1782,17 +1782,27 @@ rdb-save-incremental-fsync yes
 # the main dictionary scan
 # active-defrag-max-scan-fields 1000
 
-# Redis server/IO threads, bio threads, aof rewrite child process, and bgsave
-# child process cpu affinity list config. syntax of cpu list looks like taskset
-# command. serveral examples:
-# set redis server/io threads to cpu affinity 0,2,4,6
+# It is possible to pin different threads and processes of Redis to specific
+# CPUs in your system, in order to maximize the performances of the server.
+# This is useful both in order to pin different Redis threads in different
+# CPUs, but also in order to make sure that multiple Redis instances running
+# in the same host will be pinned to different CPUs.
+#
+# Normally you can do this using the "taskset" command, however it is also
+# possible to this via Redis configuration directly, both in Linux and FreeBSD.
+#
+# You can pin the server/IO threads, bio threads, aof rewrite child process, and
+# the bgsave child process. The syntax to specify the cpu list is the same as
+# the taskset command:
+#
+# Set redis server/io threads to cpu affinity 0,2,4,6:
 # server_cpulist 0-7:2
 #
-# set bio threads to cpu affinity 1,3
+# Set bio threads to cpu affinity 1,3:
 # bio_cpulist 1,3
 #
-# set aof rewrite child process to cpu affinity 8,9,10,11
+# Set aof rewrite child process to cpu affinity 8,9,10,11:
 # aof_rewrite_cpulist 8-11
 #
-# set bgsave child process to cpu affinity 1,10,11
+# Set bgsave child process to cpu affinity 1,10,11
 # bgsave_cpulist 1,10-11
diff --git a/src/setcpuaffinity.c b/src/setcpuaffinity.c
index 1a12795cd..dcae81c71 100644
--- a/src/setcpuaffinity.c
+++ b/src/setcpuaffinity.c
@@ -1,5 +1,5 @@
 /* ==========================================================================
- * setproctitle.c - Linux/BSD setcpuaffinity.
+ * setcpuaffinity.c - Linux/BSD setcpuaffinity.
  * --------------------------------------------------------------------------
  * Copyright (C) 2020  zhenwei pi
  *

From b062fd5235e7a66209a58be05738aff8f68f4ad8 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Mon, 4 May 2020 11:08:57 +0200
Subject: [PATCH 127/186] Fix NetBSD build by fixing redis_set_thread_title()
 support.

See #7188.
---
 src/config.h | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/config.h b/src/config.h
index a322ce354..6025d4e96 100644
--- a/src/config.h
+++ b/src/config.h
@@ -230,9 +230,12 @@ void setproctitle(const char *fmt, ...);
 #ifdef __linux__
 #define redis_set_thread_title(name) pthread_setname_np(pthread_self(), name)
 #else
-#if (defined __NetBSD__ || defined __FreeBSD__ || defined __OpenBSD__)
+#if (defined __FreeBSD__ || defined __OpenBSD__)
 #include <pthread_np.h>
 #define redis_set_thread_title(name) pthread_set_name_np(pthread_self(), name)
+#elif defined __NetBSD__
+#include <pthread.h>
+#define redis_set_thread_title(name) pthread_setname_np(pthread_self(), name, NULL)
 #else
 #if (defined __APPLE__ && defined(MAC_OS_X_VERSION_10_7))
 int pthread_setname_np(const char *name);

From a6e55c096d8246a098e4cb9b42394d9bfabc6c01 Mon Sep 17 00:00:00 2001
From: hwware <wen.hui.ware@gmail.com>
Date: Sat, 2 May 2020 19:20:44 -0400
Subject: [PATCH 128/186] Client Side Caching: Add Tracking Prefix Number Stats
 in Server Info

---
 src/server.c   | 2 ++
 src/server.h   | 1 +
 src/tracking.c | 5 +++++
 3 files changed, 8 insertions(+)

diff --git a/src/server.c b/src/server.c
index 1baa044be..f2f5c8b3f 100644
--- a/src/server.c
+++ b/src/server.c
@@ -4281,6 +4281,7 @@ sds genRedisInfoString(const char *section) {
             "active_defrag_key_misses:%lld\r\n"
             "tracking_total_keys:%lld\r\n"
             "tracking_total_items:%lld\r\n"
+            "tracking_total_prefixes:%lld\r\n"
             "unexpected_error_replies:%lld\r\n",
             server.stat_numconnections,
             server.stat_numcommands,
@@ -4311,6 +4312,7 @@ sds genRedisInfoString(const char *section) {
             server.stat_active_defrag_key_misses,
             (unsigned long long) trackingGetTotalKeys(),
             (unsigned long long) trackingGetTotalItems(),
+            (unsigned long long) trackingGetTotalPrefixes(),
             server.stat_unexpected_error_replies);
     }
 
diff --git a/src/server.h b/src/server.h
index af435b148..9c2b761c4 100644
--- a/src/server.h
+++ b/src/server.h
@@ -1695,6 +1695,7 @@ void trackingInvalidateKeysOnFlush(int dbid);
 void trackingLimitUsedSlots(void);
 uint64_t trackingGetTotalItems(void);
 uint64_t trackingGetTotalKeys(void);
+uint64_t trackingGetTotalPrefixes(void);
 void trackingBroadcastInvalidationMessages(void);
 
 /* List data type */
diff --git a/src/tracking.c b/src/tracking.c
index 48d231627..a995817e2 100644
--- a/src/tracking.c
+++ b/src/tracking.c
@@ -518,3 +518,8 @@ uint64_t trackingGetTotalKeys(void) {
     if (TrackingTable == NULL) return 0;
     return raxSize(TrackingTable);
 }
+
+uint64_t trackingGetTotalPrefixes(void) {
+    if (PrefixTable == NULL) return 0;
+    return raxSize(PrefixTable);
+}

From fc7bc3204534b4cf81c05eb14432356de5d1c536 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Tue, 5 May 2020 10:09:28 +0200
Subject: [PATCH 129/186] Fix CRC64 initialization outside the Redis server
 itself.

---
 src/crc64.c | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/crc64.c b/src/crc64.c
index 4cbc019f6..9c555e98b 100644
--- a/src/crc64.c
+++ b/src/crc64.c
@@ -29,6 +29,7 @@
 #include "crc64.h"
 #include "crcspeed.h"
 static uint64_t crc64_table[8][256] = {{0}};
+static int crc64_table_initialized = 0;
 
 #define POLY UINT64_C(0xad93d23594c935a9)
 /******************** BEGIN GENERATED PYCRC FUNCTIONS ********************/
@@ -115,10 +116,12 @@ uint64_t _crc64(uint_fast64_t crc, const void *in_data, const uint64_t len) {
 /* Initializes the 16KB lookup tables. */
 void crc64_init(void) {
     crcspeed64native_init(_crc64, crc64_table);
+    crc64_table_initialized = 1;
 }
 
 /* Compute crc64 */
 uint64_t crc64(uint64_t crc, const unsigned char *s, uint64_t l) {
+    if (!crc64_table_initialized) crc64_init();
     return crcspeed64native(crc64_table, crc, (void *) s, l);
 }
 

From 3d3861dd88651fc4a8a6d1f63ccad9b82a3a4589 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Sun, 3 May 2020 09:31:50 +0300
Subject: [PATCH 130/186] add daily github actions with libc malloc and
 valgrind

* fix memlry leaks with diskless replica short read.
* fix a few timing issues with valgrind runs
* fix issue with valgrind and watchdog schedule signal

about the valgrind WD issue:
the stack trace test in logging.tcl, has issues with valgrind:
==28808== Can't extend stack to 0x1ffeffdb38 during signal delivery for thread 1:
==28808==   too small or bad protection modes

it seems to be some valgrind bug with SA_ONSTACK.
SA_ONSTACK seems unneeded since WD is not recursive (SA_NODEFER was removed),
also, not sure if it's even valid without a call to sigaltstack()
---
 .github/workflows/daily.yml         | 48 +++++++++++++++++++++++
 src/debug.c                         |  2 +-
 src/rdb.c                           | 60 +++++++++++++++++++++--------
 tests/integration/aof.tcl           | 12 ++++++
 tests/integration/replication-3.tcl |  2 +-
 5 files changed, 106 insertions(+), 18 deletions(-)
 create mode 100644 .github/workflows/daily.yml

diff --git a/.github/workflows/daily.yml b/.github/workflows/daily.yml
new file mode 100644
index 000000000..b6a9abb68
--- /dev/null
+++ b/.github/workflows/daily.yml
@@ -0,0 +1,48 @@
+name: Daily
+
+on:
+  schedule:
+    - cron: '0 7 * * *'
+
+jobs:
+  test-jemalloc:
+    runs-on: ubuntu-latest
+    timeout-minutes: 1200
+    steps:
+    - uses: actions/checkout@v1
+    - name: make
+      run: make
+    - name: test
+      run: |
+        sudo apt-get install tcl8.5
+        ./runtest --accurate --verbose
+    - name: module api test
+      run: ./runtest-moduleapi --verbose
+
+  test-libc-malloc:
+    runs-on: ubuntu-latest
+    timeout-minutes: 1200
+    steps:
+    - uses: actions/checkout@v1
+    - name: make
+      run: make MALLOC=libc
+    - name: test
+      run: |
+        sudo apt-get install tcl8.5
+        ./runtest --accurate --verbose
+    - name: module api test
+      run: ./runtest-moduleapi --verbose
+
+  test-valgrind:
+    runs-on: ubuntu-latest
+    timeout-minutes: 14400
+    steps:
+    - uses: actions/checkout@v1
+    - name: make
+      run: make valgrind
+    - name: test
+      run: |
+        sudo apt-get install tcl8.5 valgrind -y
+        ./runtest --valgrind --verbose --clients 1
+    - name: module api test
+      run: ./runtest-moduleapi --valgrind --verbose --clients 1
diff --git a/src/debug.c b/src/debug.c
index cbb56cb71..587ff7c5d 100644
--- a/src/debug.c
+++ b/src/debug.c
@@ -1636,7 +1636,7 @@ void enableWatchdog(int period) {
         /* Watchdog was actually disabled, so we have to setup the signal
          * handler. */
         sigemptyset(&act.sa_mask);
-        act.sa_flags = SA_ONSTACK | SA_SIGINFO;
+        act.sa_flags = SA_SIGINFO;
         act.sa_sigaction = watchdogSignalHandler;
         sigaction(SIGALRM, &act, NULL);
     }
diff --git a/src/rdb.c b/src/rdb.c
index 4fc385848..6e4af3987 100644
--- a/src/rdb.c
+++ b/src/rdb.c
@@ -1441,7 +1441,10 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key) {
 
         /* Load every single element of the list */
         while(len--) {
-            if ((ele = rdbLoadEncodedStringObject(rdb)) == NULL) return NULL;
+            if ((ele = rdbLoadEncodedStringObject(rdb)) == NULL) {
+                decrRefCount(o);
+                return NULL;
+            }
             dec = getDecodedObject(ele);
             size_t len = sdslen(dec->ptr);
             quicklistPushTail(o->ptr, dec->ptr, len);
@@ -1468,8 +1471,10 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key) {
             long long llval;
             sds sdsele;
 
-            if ((sdsele = rdbGenericLoadStringObject(rdb,RDB_LOAD_SDS,NULL))
-                == NULL) return NULL;
+            if ((sdsele = rdbGenericLoadStringObject(rdb,RDB_LOAD_SDS,NULL)) == NULL) {
+                decrRefCount(o);
+                return NULL;
+            }
 
             if (o->encoding == OBJ_ENCODING_INTSET) {
                 /* Fetch integer value from element. */
@@ -1508,13 +1513,23 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key) {
             double score;
             zskiplistNode *znode;
 
-            if ((sdsele = rdbGenericLoadStringObject(rdb,RDB_LOAD_SDS,NULL))
-                == NULL) return NULL;
+            if ((sdsele = rdbGenericLoadStringObject(rdb,RDB_LOAD_SDS,NULL)) == NULL) {
+                decrRefCount(o);
+                return NULL;
+            }
 
             if (rdbtype == RDB_TYPE_ZSET_2) {
-                if (rdbLoadBinaryDoubleValue(rdb,&score) == -1) return NULL;
+                if (rdbLoadBinaryDoubleValue(rdb,&score) == -1) {
+                    decrRefCount(o);
+                    sdsfree(sdsele);
+                    return NULL;
+                }
             } else {
-                if (rdbLoadDoubleValue(rdb,&score) == -1) return NULL;
+                if (rdbLoadDoubleValue(rdb,&score) == -1) {
+                    decrRefCount(o);
+                    sdsfree(sdsele);
+                    return NULL;
+                }
             }
 
             /* Don't care about integer-encoded strings. */
@@ -1546,10 +1561,15 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key) {
         while (o->encoding == OBJ_ENCODING_ZIPLIST && len > 0) {
             len--;
             /* Load raw strings */
-            if ((field = rdbGenericLoadStringObject(rdb,RDB_LOAD_SDS,NULL))
-                == NULL) return NULL;
-            if ((value = rdbGenericLoadStringObject(rdb,RDB_LOAD_SDS,NULL))
-                == NULL) return NULL;
+            if ((field = rdbGenericLoadStringObject(rdb,RDB_LOAD_SDS,NULL)) == NULL) {
+                decrRefCount(o);
+                return NULL;
+            }
+            if ((value = rdbGenericLoadStringObject(rdb,RDB_LOAD_SDS,NULL)) == NULL) {
+                sdsfree(field);
+                decrRefCount(o);
+                return NULL;
+            }
 
             /* Add pair to ziplist */
             o->ptr = ziplistPush(o->ptr, (unsigned char*)field,
@@ -1577,10 +1597,15 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key) {
         while (o->encoding == OBJ_ENCODING_HT && len > 0) {
             len--;
             /* Load encoded strings */
-            if ((field = rdbGenericLoadStringObject(rdb,RDB_LOAD_SDS,NULL))
-                == NULL) return NULL;
-            if ((value = rdbGenericLoadStringObject(rdb,RDB_LOAD_SDS,NULL))
-                == NULL) return NULL;
+            if ((field = rdbGenericLoadStringObject(rdb,RDB_LOAD_SDS,NULL)) == NULL) {
+                decrRefCount(o);
+                return NULL;
+            }
+            if ((value = rdbGenericLoadStringObject(rdb,RDB_LOAD_SDS,NULL)) == NULL) {
+                sdsfree(field);
+                decrRefCount(o);
+                return NULL;
+            }
 
             /* Add pair to hash table */
             ret = dictAdd((dict*)o->ptr, field, value);
@@ -1600,7 +1625,10 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key) {
         while (len--) {
             unsigned char *zl =
                 rdbGenericLoadStringObject(rdb,RDB_LOAD_PLAIN,NULL);
-            if (zl == NULL) return NULL;
+            if (zl == NULL) {
+                decrRefCount(o);
+                return NULL;
+            }
             quicklistAppendZiplist(o->ptr, zl);
         }
     } else if (rdbtype == RDB_TYPE_HASH_ZIPMAP  ||
diff --git a/tests/integration/aof.tcl b/tests/integration/aof.tcl
index 2734de7f1..b82c87d71 100644
--- a/tests/integration/aof.tcl
+++ b/tests/integration/aof.tcl
@@ -54,6 +54,12 @@ tags {"aof"} {
 
         set client [redis [dict get $srv host] [dict get $srv port] 0 $::tls]
 
+        wait_for_condition 50 100 {
+            [catch {$client ping} e] == 0
+        } else {
+            fail "Loading DB is taking too much time."
+        }
+
         test "Truncated AOF loaded: we expect foo to be equal to 5" {
             assert {[$client get foo] eq "5"}
         }
@@ -71,6 +77,12 @@ tags {"aof"} {
 
         set client [redis [dict get $srv host] [dict get $srv port] 0 $::tls]
 
+        wait_for_condition 50 100 {
+            [catch {$client ping} e] == 0
+        } else {
+            fail "Loading DB is taking too much time."
+        }
+
         test "Truncated AOF loaded: we expect foo to be equal to 6 now" {
             assert {[$client get foo] eq "6"}
         }
diff --git a/tests/integration/replication-3.tcl b/tests/integration/replication-3.tcl
index 198e698f2..43eb53538 100644
--- a/tests/integration/replication-3.tcl
+++ b/tests/integration/replication-3.tcl
@@ -118,7 +118,7 @@ start_server {tags {"repl"}} {
             # correctly the RDB file: such file will contain "lua" AUX
             # sections with scripts already in the memory of the master.
 
-            wait_for_condition 50 100 {
+            wait_for_condition 500 100 {
                 [s -1 master_link_status] eq {up}
             } else {
                 fail "Replication not started."

From 75addb4fe23176199de4d8946301a4086c3e75e5 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Fri, 24 Apr 2020 17:20:28 +0300
Subject: [PATCH 131/186] optimize memory usage of deferred replies - fixed

When deffered reply is added the previous reply node cannot be used so
all the extra space we allocated in it is wasted. in case someone uses
deffered replies in a loop, each time adding a small reply, each of
these reply nodes (the small string reply) would have consumed a 16k
block.
now when we add anther diferred reply node, we trim the unused portion
of the previous reply block.

see #7123

cherry picked from commit fb732f7a944a4d4c90bb7375cb6030e88211f5aa
with fix to handle a crash with LIBC allocator, which apparently can
return the same pointer despite changing it's size.
i.e. shrinking an allocation of 16k into 56 bytes without changing the
pointer.
---
 src/networking.c | 29 +++++++++++++++++++++++++++++
 1 file changed, 29 insertions(+)

diff --git a/src/networking.c b/src/networking.c
index c4a277e0a..d62533e3e 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -436,6 +436,34 @@ void addReplyStatusFormat(client *c, const char *fmt, ...) {
     sdsfree(s);
 }
 
+/* Sometimes we are forced to create a new reply node, and we can't append to
+ * the previous one, when that happens, we wanna try to trim the unused space
+ * at the end of the last reply node which we won't use anymore. */
+void trimReplyUnusedTailSpace(client *c) {
+    listNode *ln = listLast(c->reply);
+    clientReplyBlock *tail = ln? listNodeValue(ln): NULL;
+
+    /* Note that 'tail' may be NULL even if we have a tail node, becuase when
+     * addDeferredMultiBulkLength() is used */
+    if (!tail) return;
+
+    /* We only try to trim the space is relatively high (more than a 1/4 of the
+     * allocation), otherwise there's a high chance realloc will NOP.
+     * Also, to avoid large memmove which happens as part of realloc, we only do
+     * that if the used part is small.  */
+    if (tail->size - tail->used > tail->size / 4 &&
+        tail->used < PROTO_REPLY_CHUNK_BYTES)
+    {
+        size_t old_size = tail->size;
+        tail = zrealloc(tail, tail->used + sizeof(clientReplyBlock));
+        /* take over the allocation's internal fragmentation (at least for
+         * memory usage tracking) */
+        tail->size = zmalloc_usable(tail) - sizeof(clientReplyBlock);
+        c->reply_bytes += tail->size - old_size;
+        listNodeValue(ln) = tail;
+    }
+}
+
 /* Adds an empty object to the reply list that will contain the multi bulk
  * length, which is not known when this function is called. */
 void *addReplyDeferredLen(client *c) {
@@ -443,6 +471,7 @@ void *addReplyDeferredLen(client *c) {
      * ready to be sent, since we are sure that before returning to the
      * event loop setDeferredAggregateLen() will be called. */
     if (prepareClientToWrite(c) != C_OK) return NULL;
+    trimReplyUnusedTailSpace(c);
     listAddNodeTail(c->reply,NULL); /* NULL is our placeholder. */
     return listLast(c->reply);
 }

From ac316d8cc57cef74c520558b024cb8c0babd7e14 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Tue, 5 May 2020 10:20:48 +0200
Subject: [PATCH 132/186] Move CRC64 initialization in main().

---
 src/crc64.c  | 3 ---
 src/server.c | 2 +-
 2 files changed, 1 insertion(+), 4 deletions(-)

diff --git a/src/crc64.c b/src/crc64.c
index 9c555e98b..4cbc019f6 100644
--- a/src/crc64.c
+++ b/src/crc64.c
@@ -29,7 +29,6 @@
 #include "crc64.h"
 #include "crcspeed.h"
 static uint64_t crc64_table[8][256] = {{0}};
-static int crc64_table_initialized = 0;
 
 #define POLY UINT64_C(0xad93d23594c935a9)
 /******************** BEGIN GENERATED PYCRC FUNCTIONS ********************/
@@ -116,12 +115,10 @@ uint64_t _crc64(uint_fast64_t crc, const void *in_data, const uint64_t len) {
 /* Initializes the 16KB lookup tables. */
 void crc64_init(void) {
     crcspeed64native_init(_crc64, crc64_table);
-    crc64_table_initialized = 1;
 }
 
 /* Compute crc64 */
 uint64_t crc64(uint64_t crc, const unsigned char *s, uint64_t l) {
-    if (!crc64_table_initialized) crc64_init();
     return crcspeed64native(crc64_table, crc, (void *) s, l);
 }
 
diff --git a/src/server.c b/src/server.c
index f2f5c8b3f..416dcd747 100644
--- a/src/server.c
+++ b/src/server.c
@@ -2899,7 +2899,6 @@ void initServer(void) {
     scriptingInit(1);
     slowlogInit();
     latencyMonitorInit();
-    crc64_init();
 }
 
 /* Some steps in server initialization need to be done last (after modules
@@ -4963,6 +4962,7 @@ int main(int argc, char **argv) {
     zmalloc_set_oom_handler(redisOutOfMemoryHandler);
     srand(time(NULL)^getpid());
     gettimeofday(&tv,NULL);
+    crc64_init();
 
     uint8_t hashseed[16];
     getRandomBytes(hashseed,sizeof(hashseed));

From c57d9146f41f4b661d9d2cb48b83b3abc757ba0e Mon Sep 17 00:00:00 2001
From: Deliang Yang <yang623601391@gmail.com>
Date: Mon, 4 May 2020 01:57:38 +0800
Subject: [PATCH 133/186] reformat code

---
 src/ziplist.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/ziplist.c b/src/ziplist.c
index ef40d6aa2..ddae0d96f 100644
--- a/src/ziplist.c
+++ b/src/ziplist.c
@@ -440,7 +440,7 @@ unsigned int zipStorePrevEntryLength(unsigned char *p, unsigned int len) {
     if ((prevlensize) == 1) {                                                  \
         (prevlen) = (ptr)[0];                                                  \
     } else if ((prevlensize) == 5) {                                           \
-        assert(sizeof((prevlen)) == 4);                                    \
+        assert(sizeof((prevlen)) == 4);                                        \
         memcpy(&(prevlen), ((char*)(ptr)) + 1, 4);                             \
         memrev32ifbe(&prevlen);                                                \
     }                                                                          \

From 3a441c7d95c8757d2ab12ff730cc7ca8899538c9 Mon Sep 17 00:00:00 2001
From: Guy Benoish <guy.benoish@redislabs.com>
Date: Sun, 3 May 2020 16:49:45 +0300
Subject: [PATCH 134/186] XPENDING should not update consumer's seen-time

Same goes for XGROUP DELCONSUMER (But in this case, it doesn't
have any visible effect)
---
 src/blocked.c  |  7 ++++---
 src/rdb.c      |  4 ++--
 src/stream.h   |  7 ++++++-
 src/t_stream.c | 33 ++++++++++++++++++++-------------
 4 files changed, 32 insertions(+), 19 deletions(-)

diff --git a/src/blocked.c b/src/blocked.c
index 045369e93..92f1cee65 100644
--- a/src/blocked.c
+++ b/src/blocked.c
@@ -371,9 +371,10 @@ void serveClientsBlockedOnStreamKey(robj *o, readyList *rl) {
                 int noack = 0;
 
                 if (group) {
-                    consumer = streamLookupConsumer(group,
-                               receiver->bpop.xread_consumer->ptr,
-                               1);
+                    consumer =
+                        streamLookupConsumer(group,
+                                             receiver->bpop.xread_consumer->ptr,
+                                             SLC_NONE);
                     noack = receiver->bpop.xread_group_noack;
                 }
 
diff --git a/src/rdb.c b/src/rdb.c
index 6e4af3987..c882efcb4 100644
--- a/src/rdb.c
+++ b/src/rdb.c
@@ -1851,8 +1851,8 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key) {
                     decrRefCount(o);
                     return NULL;
                 }
-                streamConsumer *consumer = streamLookupConsumer(cgroup,cname,
-                                           1);
+                streamConsumer *consumer =
+                    streamLookupConsumer(cgroup,cname,SLC_NONE);
                 sdsfree(cname);
                 consumer->seen_time = rdbLoadMillisecondTime(rdb,RDB_VERSION);
                 if (rioGetReadError(rdb)) {
diff --git a/src/stream.h b/src/stream.h
index b69073994..0d3bf63fc 100644
--- a/src/stream.h
+++ b/src/stream.h
@@ -96,6 +96,11 @@ typedef struct streamPropInfo {
 /* Prototypes of exported APIs. */
 struct client;
 
+/* Flags for streamLookupConsumer */
+#define SLC_NONE      0
+#define SLC_NOCREAT   (1<<0) /* Do not create the consumer if it doesn't exist */
+#define SLC_NOREFRESH (1<<1) /* Do not update consumer's seen-time */
+
 stream *streamNew(void);
 void freeStream(stream *s);
 unsigned long streamLength(const robj *subject);
@@ -105,7 +110,7 @@ int streamIteratorGetID(streamIterator *si, streamID *id, int64_t *numfields);
 void streamIteratorGetField(streamIterator *si, unsigned char **fieldptr, unsigned char **valueptr, int64_t *fieldlen, int64_t *valuelen);
 void streamIteratorStop(streamIterator *si);
 streamCG *streamLookupCG(stream *s, sds groupname);
-streamConsumer *streamLookupConsumer(streamCG *cg, sds name, int create);
+streamConsumer *streamLookupConsumer(streamCG *cg, sds name, int flags);
 streamCG *streamCreateCG(stream *s, char *name, size_t namelen, streamID *id);
 streamNACK *streamCreateNACK(streamConsumer *consumer);
 void streamDecodeID(void *buf, streamID *id);
diff --git a/src/t_stream.c b/src/t_stream.c
index 5c1b9a523..676ddd9bb 100644
--- a/src/t_stream.c
+++ b/src/t_stream.c
@@ -1570,7 +1570,8 @@ void xreadCommand(client *c) {
             addReplyBulk(c,c->argv[streams_arg+i]);
             streamConsumer *consumer = NULL;
             if (groups) consumer = streamLookupConsumer(groups[i],
-                                                        consumername->ptr,1);
+                                                        consumername->ptr,
+                                                        SLC_NONE);
             streamPropInfo spi = {c->argv[i+streams_arg],groupname};
             int flags = 0;
             if (noack) flags |= STREAM_RWR_NOACK;
@@ -1706,7 +1707,9 @@ streamCG *streamLookupCG(stream *s, sds groupname) {
  * consumer does not exist it is automatically created as a side effect
  * of calling this function, otherwise its last seen time is updated and
  * the existing consumer reference returned. */
-streamConsumer *streamLookupConsumer(streamCG *cg, sds name, int create) {
+streamConsumer *streamLookupConsumer(streamCG *cg, sds name, int flags) {
+    int create = !(flags & SLC_NOCREAT);
+    int refresh = !(flags & SLC_NOREFRESH);
     streamConsumer *consumer = raxFind(cg->consumers,(unsigned char*)name,
                                sdslen(name));
     if (consumer == raxNotFound) {
@@ -1717,7 +1720,7 @@ streamConsumer *streamLookupConsumer(streamCG *cg, sds name, int create) {
         raxInsert(cg->consumers,(unsigned char*)name,sdslen(name),
                   consumer,NULL);
     }
-    consumer->seen_time = mstime();
+    if (refresh) consumer->seen_time = mstime();
     return consumer;
 }
 
@@ -1725,7 +1728,8 @@ streamConsumer *streamLookupConsumer(streamCG *cg, sds name, int create) {
  * may have pending messages: they are removed from the PEL, and the number
  * of pending messages "lost" is returned. */
 uint64_t streamDelConsumer(streamCG *cg, sds name) {
-    streamConsumer *consumer = streamLookupConsumer(cg,name,0);
+    streamConsumer *consumer =
+        streamLookupConsumer(cg,name,SLC_NOCREAT|SLC_NOREFRESH);
     if (consumer == NULL) return 0;
 
     uint64_t retval = raxSize(consumer->pel);
@@ -2068,15 +2072,18 @@ void xpendingCommand(client *c) {
     }
     /* XPENDING <key> <group> <start> <stop> <count> [<consumer>] variant. */
     else {
-        streamConsumer *consumer = consumername ?
-                                streamLookupConsumer(group,consumername->ptr,0):
-                                NULL;
+        streamConsumer *consumer = NULL;
+        if (consumername) {
+            consumer = streamLookupConsumer(group,
+                                            consumername->ptr,
+                                            SLC_NOCREAT|SLC_NOREFRESH);
 
-        /* If a consumer name was mentioned but it does not exist, we can
-         * just return an empty array. */
-        if (consumername && consumer == NULL) {
-            addReplyArrayLen(c,0);
-            return;
+            /* If a consumer name was mentioned but it does not exist, we can
+             * just return an empty array. */
+            if (consumer == NULL) {
+                addReplyArrayLen(c,0);
+                return;
+            }
         }
 
         rax *pel = consumer ? consumer->pel : group->pel;
@@ -2338,7 +2345,7 @@ void xclaimCommand(client *c) {
                 raxRemove(nack->consumer->pel,buf,sizeof(buf),NULL);
             /* Update the consumer and idle time. */
             if (consumer == NULL)
-                consumer = streamLookupConsumer(group,c->argv[3]->ptr,1);
+                consumer = streamLookupConsumer(group,c->argv[3]->ptr,SLC_NONE);
             nack->consumer = consumer;
             nack->delivery_time = deliverytime;
             /* Set the delivery attempts counter if given, otherwise

From 897a360d060db7acb6d9898b486a067cd895d232 Mon Sep 17 00:00:00 2001
From: Muhammad Zahalqa <m@tryfinally.com>
Date: Mon, 4 May 2020 12:36:01 +0300
Subject: [PATCH 135/186] Fix compiler warnings on function rev(unsigned long)

---
 src/dict.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/dict.c b/src/dict.c
index ac6f8cfde..57619c280 100644
--- a/src/dict.c
+++ b/src/dict.c
@@ -766,9 +766,9 @@ dictEntry *dictGetFairRandomKey(dict *d) {
 /* Function to reverse bits. Algorithm from:
  * http://graphics.stanford.edu/~seander/bithacks.html#ReverseParallel */
 static unsigned long rev(unsigned long v) {
-    unsigned long s = 8 * sizeof(v); // bit size; must be power of 2
-    unsigned long mask = ~0;
-    while ((s >>= 1) > 0) {
+    unsigned long s = CHAR_BIT * sizeof(v); // bit size; must be power of 2
+    unsigned long mask = ~0UL;
+    while ((s >>= 1) > 0UL) {
         mask ^= (mask << s);
         v = ((v >> s) & mask) | ((v << s) & ~mask);
     }

From d1af82a8838764469e4536c984a61379f33e7260 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Tue, 5 May 2020 10:27:27 +0200
Subject: [PATCH 136/186] Drop not needed part from #7194.

---
 src/dict.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/dict.c b/src/dict.c
index 57619c280..13c185253 100644
--- a/src/dict.c
+++ b/src/dict.c
@@ -768,7 +768,7 @@ dictEntry *dictGetFairRandomKey(dict *d) {
 static unsigned long rev(unsigned long v) {
     unsigned long s = CHAR_BIT * sizeof(v); // bit size; must be power of 2
     unsigned long mask = ~0UL;
-    while ((s >>= 1) > 0UL) {
+    while ((s >>= 1) > 0) {
         mask ^= (mask << s);
         v = ((v >> s) & mask) | ((v << s) & ~mask);
     }

From 1e561cfaaf0a867f986569147a5cb9dd2cded1af Mon Sep 17 00:00:00 2001
From: Benjamin Sergeant <bsergean@gmail.com>
Date: Mon, 4 May 2020 08:09:21 -0700
Subject: [PATCH 137/186] Add --user argument to redis-benchmark.c (ACL)

---
 src/redis-benchmark.c | 17 +++++++++++++++--
 1 file changed, 15 insertions(+), 2 deletions(-)

diff --git a/src/redis-benchmark.c b/src/redis-benchmark.c
index 2df41580b..77daf981c 100644
--- a/src/redis-benchmark.c
+++ b/src/redis-benchmark.c
@@ -94,6 +94,7 @@ static struct config {
     sds dbnumstr;
     char *tests;
     char *auth;
+    const char *user;
     int precision;
     int num_threads;
     struct benchmarkThread **threads;
@@ -258,7 +259,10 @@ static redisConfig *getRedisConfig(const char *ip, int port,
 
     if(config.auth) {
         void *authReply = NULL;
-        redisAppendCommand(c, "AUTH %s", config.auth);
+        if (config.user == NULL)
+            redisAppendCommand(c, "AUTH %s", config.auth);
+        else
+            redisAppendCommand(c, "AUTH %s %s", config.user, config.auth);
         if (REDIS_OK != redisGetReply(c, &authReply)) goto fail;
         if (reply) freeReplyObject(reply);
         reply = ((redisReply *) authReply);
@@ -628,7 +632,12 @@ static client createClient(char *cmd, size_t len, client from, int thread_id) {
     c->prefix_pending = 0;
     if (config.auth) {
         char *buf = NULL;
-        int len = redisFormatCommand(&buf, "AUTH %s", config.auth);
+        int len;
+        if (config.user == NULL)
+            len = redisFormatCommand(&buf, "AUTH %s", config.auth);
+        else
+            len = redisFormatCommand(&buf, "AUTH %s %s",
+                                     config.user, config.auth);
         c->obuf = sdscatlen(c->obuf, buf, len);
         free(buf);
         c->prefix_pending++;
@@ -1299,6 +1308,9 @@ int parseOptions(int argc, const char **argv) {
         } else if (!strcmp(argv[i],"-a") ) {
             if (lastarg) goto invalid;
             config.auth = strdup(argv[++i]);
+        } else if (!strcmp(argv[i],"--user")) {
+            if (lastarg) goto invalid;
+            config.user = argv[++i];
         } else if (!strcmp(argv[i],"-d")) {
             if (lastarg) goto invalid;
             config.datasize = atoi(argv[++i]);
@@ -1385,6 +1397,7 @@ usage:
 " -p <port>          Server port (default 6379)\n"
 " -s <socket>        Server socket (overrides host and port)\n"
 " -a <password>      Password for Redis Auth\n"
+" --user <username>  Used to send ACL style 'AUTH username pass'. Needs -a.\n"
 " -c <clients>       Number of parallel connections (default 50)\n"
 " -n <requests>      Total number of requests (default 100000)\n"
 " -d <size>          Data size of SET/GET value in bytes (default 3)\n"

From 8a9c84f4a5737499bd630fb8de6c7359637d3987 Mon Sep 17 00:00:00 2001
From: hwware <wen.hui.ware@gmail.com>
Date: Tue, 5 May 2020 23:35:08 -0400
Subject: [PATCH 138/186] add include guard for lolwut.h

---
 src/lolwut.h | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/src/lolwut.h b/src/lolwut.h
index 38c0de423..682d00531 100644
--- a/src/lolwut.h
+++ b/src/lolwut.h
@@ -34,6 +34,10 @@
 /* This represents a very simple generic canvas in order to draw stuff.
  * It's up to each LOLWUT versions to translate what they draw to the
  * screen, depending on the result to accomplish. */
+
+#ifndef __LOLWUT_H
+#define __LOLWUT_H
+
 typedef struct lwCanvas {
     int width;
     int height;
@@ -47,3 +51,5 @@ void lwDrawPixel(lwCanvas *canvas, int x, int y, int color);
 int lwGetPixel(lwCanvas *canvas, int x, int y);
 void lwDrawLine(lwCanvas *canvas, int x1, int y1, int x2, int y2, int color);
 void lwDrawSquare(lwCanvas *canvas, int x, int y, float size, float angle, int color);
+
+#endif

From e48c37316e281aaa137624bd32abde2ab47858ea Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Tue, 5 May 2020 13:18:53 +0200
Subject: [PATCH 139/186] Test: --dont-clean should do first cleanup.

---
 tests/test_helper.tcl | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/tests/test_helper.tcl b/tests/test_helper.tcl
index 3eee1aeb7..de0a64728 100644
--- a/tests/test_helper.tcl
+++ b/tests/test_helper.tcl
@@ -216,9 +216,6 @@ proc run_solo {name code} {
 }
 
 proc cleanup {} {
-    if {$::dont_clean} {
-        return
-    }
     if {!$::quiet} {puts -nonewline "Cleanup: may take some time... "}
     flush stdout
     catch {exec rm -rf {*}[glob tests/tmp/redis.conf.*]}
@@ -456,11 +453,11 @@ proc the_end {} {
         foreach failed $::failed_tests {
             puts "*** $failed"
         }
-        cleanup
+        if {!$::dont_clean} cleanup
         exit 1
     } else {
         puts "\n[colorstr bold-white {\o/}] [colorstr bold-green {All tests passed without errors!}]\n"
-        cleanup
+        if {!$::dont_clean} cleanup
         exit 0
     }
 }

From c7edffbd56c7a2c95c2483485fe86dcb43077fa4 Mon Sep 17 00:00:00 2001
From: hwware <wen.hui.ware@gmail.com>
Date: Wed, 6 May 2020 01:07:17 -0400
Subject: [PATCH 140/186] add jemalloc-bg-thread config in redis conf

---
 redis.conf | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/redis.conf b/redis.conf
index bed906058..bab7cff47 100644
--- a/redis.conf
+++ b/redis.conf
@@ -1782,6 +1782,9 @@ rdb-save-incremental-fsync yes
 # the main dictionary scan
 # active-defrag-max-scan-fields 1000
 
+# Jemalloc background thread for purging will be enabled by default
+jemalloc-bg-thread yes
+
 # It is possible to pin different threads and processes of Redis to specific
 # CPUs in your system, in order to maximize the performances of the server.
 # This is useful both in order to pin different Redis threads in different

From ec1e106ec5c4ef9fd71d3ab3a6137012120058f4 Mon Sep 17 00:00:00 2001
From: Titouan Christophe <titouan.christophe@railnova.eu>
Date: Mon, 4 May 2020 14:11:43 +0200
Subject: [PATCH 141/186] make struct user anonymous (only typedefed)

This works because this struct is never referenced by its name,
but always by its type.

This prevents a conflict with struct user from <sys/user.h>
when compiling against uclibc.

Signed-off-by: Titouan Christophe <titouan.christophe@railnova.eu>
---
 src/server.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/server.h b/src/server.h
index 9c2b761c4..59cf1370e 100644
--- a/src/server.h
+++ b/src/server.h
@@ -732,7 +732,7 @@ typedef struct readyList {
                                            no AUTH is needed, and every
                                            connection is immediately
                                            authenticated. */
-typedef struct user {
+typedef struct {
     sds name;       /* The username as an SDS string. */
     uint64_t flags; /* See USER_FLAG_* */
 

From 24e12641d579d31f5a0fd9b8ed2ccf8e10806596 Mon Sep 17 00:00:00 2001
From: Brad Dunbar <dunbarb2@gmail.com>
Date: Tue, 5 May 2020 09:57:01 -0400
Subject: [PATCH 142/186] Remove unreachable branch.

---
 src/util.c | 2 --
 1 file changed, 2 deletions(-)

diff --git a/src/util.c b/src/util.c
index d173f776f..537c9313c 100644
--- a/src/util.c
+++ b/src/util.c
@@ -67,8 +67,6 @@ int stringmatchlen(const char *pattern, int patternLen,
             return 0; /* no match */
             break;
         case '?':
-            if (stringLen == 0)
-                return 0; /* no match */
             string++;
             stringLen--;
             break;

From 84d9766d6a4bc04a642bfec3b82386483dd452c5 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Tue, 5 May 2020 13:40:33 +0200
Subject: [PATCH 143/186] Dump recent backlog on master query generating
 errors.

---
 src/networking.c | 29 +++++++++++++++++++++++++++++
 1 file changed, 29 insertions(+)

diff --git a/src/networking.c b/src/networking.c
index fd1159e06..75c0c16b1 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -393,6 +393,35 @@ void addReplyErrorLength(client *c, const char *s, size_t len) {
         serverLog(LL_WARNING,"== CRITICAL == This %s is sending an error "
                              "to its %s: '%s' after processing the command "
                              "'%s'", from, to, s, cmdname);
+        if (ctype == CLIENT_TYPE_MASTER && server.repl_backlog &&
+            server.repl_backlog_histlen > 0)
+        {
+            long long dumplen = 256;
+            if (server.repl_backlog_histlen < dumplen)
+                dumplen = server.repl_backlog_histlen;
+
+            /* Identify the first byte to dump. */
+            long long idx =
+              (server.repl_backlog_idx + (server.repl_backlog_size - dumplen)) %
+               server.repl_backlog_size;
+
+            /* Scan the circular buffer to collect 'dumplen' bytes. */
+            sds dump = sdsempty();
+            while(dumplen) {
+                long long thislen =
+                    ((server.repl_backlog_size - idx) < dumplen) ?
+                    (server.repl_backlog_size - idx) : dumplen;
+
+                dump = sdscatrepr(dump,server.repl_backlog+idx,thislen);
+                dumplen -= thislen;
+                idx = 0;
+            }
+
+            /* Finally log such bytes: this is vital debugging info to
+             * understand what happened. */
+            serverLog(LL_WARNING,"Latest backlog is: '%s'", dump);
+            sdsfree(dump);
+        }
         server.stat_unexpected_error_replies++;
     }
 }

From 3672875b4cb7bca525bbc2da42f49153fcb7e0eb Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Wed, 6 May 2020 16:18:21 +0200
Subject: [PATCH 144/186] stringmatchlen() should not expect null terminated
 strings.

---
 src/util.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/util.c b/src/util.c
index 537c9313c..0d48f5701 100644
--- a/src/util.c
+++ b/src/util.c
@@ -51,7 +51,7 @@ int stringmatchlen(const char *pattern, int patternLen,
     while(patternLen && stringLen) {
         switch(pattern[0]) {
         case '*':
-            while (pattern[1] == '*') {
+            while (patternLen && pattern[1] == '*') {
                 pattern++;
                 patternLen--;
             }
@@ -94,7 +94,7 @@ int stringmatchlen(const char *pattern, int patternLen,
                     pattern--;
                     patternLen++;
                     break;
-                } else if (pattern[1] == '-' && patternLen >= 3) {
+                } else if (patternLen >= 3 && pattern[1] == '-') {
                     int start = pattern[0];
                     int end = pattern[2];
                     int c = string[0];

From cb683a84f7a9d530bec629f2b656e7d0842a0f75 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Tue, 5 May 2020 15:50:00 +0200
Subject: [PATCH 145/186] Don't propagate spurious MULTI on DEBUG LOADAOF.

---
 src/multi.c  | 5 ++++-
 src/server.c | 4 ++--
 2 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/src/multi.c b/src/multi.c
index cbbd2c513..a331a6240 100644
--- a/src/multi.c
+++ b/src/multi.c
@@ -172,7 +172,10 @@ void execCommand(client *c) {
          * This way we'll deliver the MULTI/..../EXEC block as a whole and
          * both the AOF and the replication link will have the same consistency
          * and atomicity guarantees. */
-        if (!must_propagate && !(c->cmd->flags & (CMD_READONLY|CMD_ADMIN))) {
+        if (!must_propagate &&
+            !server.loading &&
+            !(c->cmd->flags & (CMD_READONLY|CMD_ADMIN)))
+        {
             execCommandPropagateMulti(c);
             must_propagate = 1;
         }
diff --git a/src/server.c b/src/server.c
index 416dcd747..e2b4b6f3d 100644
--- a/src/server.c
+++ b/src/server.c
@@ -3204,8 +3204,8 @@ void call(client *c, int flags) {
 
     server.fixed_time_expire++;
 
-    /* Sent the command to clients in MONITOR mode, only if the commands are
-     * not generated from reading an AOF. */
+    /* Send the command to clients in MONITOR mode if applicable.
+     * Administrative commands are considered too dangerous to be shown. */
     if (listLength(server.monitors) &&
         !server.loading &&
         !(c->cmd->flags & (CMD_SKIP_MONITOR|CMD_ADMIN)))

From 960186a71fee12b0e64b109c7b2066d2dae0ffe5 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Fri, 8 May 2020 11:38:07 +0200
Subject: [PATCH 146/186] Cluster: introduce data_received field.

We want to send pings and pongs at specific intervals, since our packets
also contain information about the configuration of the cluster and are
used for gossip. However since our cluster bus is used in a mixed way
for data (such as Pub/Sub or modules cluster messages) and metadata,
sometimes a very busy channel may delay the reception of pong packets.
So after discussing it in #7216, this commit introduces a new field that
is not exposed in the cluster, is only an internal information about
the last time we received any data from a given node: we use this field
in order to avoid detecting failures, claiming data reception of new
data from the node is a proof of liveness.
---
 src/cluster.c | 36 ++++++++++++++++++++++++++----------
 src/cluster.h |  1 +
 2 files changed, 27 insertions(+), 10 deletions(-)

diff --git a/src/cluster.c b/src/cluster.c
index d8f83f08d..78524bd34 100644
--- a/src/cluster.c
+++ b/src/cluster.c
@@ -749,6 +749,7 @@ clusterNode *createClusterNode(char *nodename, int flags) {
     node->slaves = NULL;
     node->slaveof = NULL;
     node->ping_sent = node->pong_received = 0;
+    node->data_received = 0;
     node->fail_time = 0;
     node->link = NULL;
     memset(node->ip,0,sizeof(node->ip));
@@ -1678,6 +1679,7 @@ int clusterProcessPacket(clusterLink *link) {
     clusterMsg *hdr = (clusterMsg*) link->rcvbuf;
     uint32_t totlen = ntohl(hdr->totlen);
     uint16_t type = ntohs(hdr->type);
+    mstime_t now = mstime();
 
     if (type < CLUSTERMSG_TYPE_COUNT)
         server.cluster->stats_bus_messages_received[type]++;
@@ -1741,6 +1743,13 @@ int clusterProcessPacket(clusterLink *link) {
 
     /* Check if the sender is a known node. */
     sender = clusterLookupNode(hdr->sender);
+
+    /* Update the last time we saw any data from this node. We
+     * use this in order to avoid detecting a timeout from a node that
+     * is just sending a lot of data in the cluster bus, for instance
+     * because of Pub/Sub. */
+    if (sender) sender->data_received = now;
+
     if (sender && !nodeInHandshake(sender)) {
         /* Update our curretEpoch if we see a newer epoch in the cluster. */
         senderCurrentEpoch = ntohu64(hdr->currentEpoch);
@@ -1755,7 +1764,7 @@ int clusterProcessPacket(clusterLink *link) {
         }
         /* Update the replication offset info for this node. */
         sender->repl_offset = ntohu64(hdr->offset);
-        sender->repl_offset_time = mstime();
+        sender->repl_offset_time = now;
         /* If we are a slave performing a manual failover and our master
          * sent its offset while already paused, populate the MF state. */
         if (server.cluster->mf_end &&
@@ -1869,7 +1878,7 @@ int clusterProcessPacket(clusterLink *link) {
                  * address. */
                 serverLog(LL_DEBUG,"PONG contains mismatching sender ID. About node %.40s added %d ms ago, having flags %d",
                     link->node->name,
-                    (int)(mstime()-(link->node->ctime)),
+                    (int)(now-(link->node->ctime)),
                     link->node->flags);
                 link->node->flags |= CLUSTER_NODE_NOADDR;
                 link->node->ip[0] = '\0';
@@ -1904,7 +1913,7 @@ int clusterProcessPacket(clusterLink *link) {
 
         /* Update our info about the node */
         if (link->node && type == CLUSTERMSG_TYPE_PONG) {
-            link->node->pong_received = mstime();
+            link->node->pong_received = now;
             link->node->ping_sent = 0;
 
             /* The PFAIL condition can be reversed without external
@@ -2051,7 +2060,7 @@ int clusterProcessPacket(clusterLink *link) {
                     "FAIL message received from %.40s about %.40s",
                     hdr->sender, hdr->data.fail.about.nodename);
                 failing->flags |= CLUSTER_NODE_FAIL;
-                failing->fail_time = mstime();
+                failing->fail_time = now;
                 failing->flags &= ~CLUSTER_NODE_PFAIL;
                 clusterDoBeforeSleep(CLUSTER_TODO_SAVE_CONFIG|
                                      CLUSTER_TODO_UPDATE_STATE);
@@ -2104,9 +2113,9 @@ int clusterProcessPacket(clusterLink *link) {
         /* Manual failover requested from slaves. Initialize the state
          * accordingly. */
         resetManualFailover();
-        server.cluster->mf_end = mstime() + CLUSTER_MF_TIMEOUT;
+        server.cluster->mf_end = now + CLUSTER_MF_TIMEOUT;
         server.cluster->mf_slave = sender;
-        pauseClients(mstime()+(CLUSTER_MF_TIMEOUT*CLUSTER_MF_PAUSE_MULT));
+        pauseClients(now+(CLUSTER_MF_TIMEOUT*CLUSTER_MF_PAUSE_MULT));
         serverLog(LL_WARNING,"Manual failover requested by replica %.40s.",
             sender->name);
     } else if (type == CLUSTERMSG_TYPE_UPDATE) {
@@ -3529,7 +3538,6 @@ void clusterCron(void) {
     while((de = dictNext(di)) != NULL) {
         clusterNode *node = dictGetVal(de);
         now = mstime(); /* Use an updated time at every iteration. */
-        mstime_t delay;
 
         if (node->flags &
             (CLUSTER_NODE_MYSELF|CLUSTER_NODE_NOADDR|CLUSTER_NODE_HANDSHAKE))
@@ -3553,7 +3561,7 @@ void clusterCron(void) {
                 this_slaves = okslaves;
         }
 
-        /* If we are waiting for the PONG more than half the cluster
+        /* If we are not receiving any data for more than half the cluster
          * timeout, reconnect the link: maybe there is a connection
          * issue even if the node is alive. */
         if (node->link && /* is connected */
@@ -3562,7 +3570,9 @@ void clusterCron(void) {
             node->ping_sent && /* we already sent a ping */
             node->pong_received < node->ping_sent && /* still waiting pong */
             /* and we are waiting for the pong more than timeout/2 */
-            now - node->ping_sent > server.cluster_node_timeout/2)
+            now - node->ping_sent > server.cluster_node_timeout/2 &&
+            /* and in such interval we are not seeing any traffic at all. */
+            now - node->data_received > server.cluster_node_timeout/2)
         {
             /* Disconnect the link, it will be reconnected automatically. */
             freeClusterLink(node->link);
@@ -3597,7 +3607,13 @@ void clusterCron(void) {
         /* Compute the delay of the PONG. Note that if we already received
          * the PONG, then node->ping_sent is zero, so can't reach this
          * code at all. */
-        delay = now - node->ping_sent;
+        mstime_t delay = now - node->ping_sent;
+
+        /* We consider every incoming data as proof of liveness, since
+         * our cluster bus link is also used for data: under heavy data
+         * load pong delays are possible. */
+        mstime_t data_delay = now - node->data_received;
+        if (data_delay < delay) delay = data_delay;
 
         if (delay > server.cluster_node_timeout) {
             /* Timeout reached. Set the node as possibly failing if it is
diff --git a/src/cluster.h b/src/cluster.h
index 35fc0cbfa..d3af4a355 100644
--- a/src/cluster.h
+++ b/src/cluster.h
@@ -124,6 +124,7 @@ typedef struct clusterNode {
                                     tables. */
     mstime_t ping_sent;      /* Unix time we sent latest ping */
     mstime_t pong_received;  /* Unix time we received the pong */
+    mstime_t data_received;  /* Unix time we received any data */
     mstime_t fail_time;      /* Unix time when FAIL flag was set */
     mstime_t voted_time;     /* Last time we voted for a slave of this master */
     mstime_t repl_offset_time;  /* Unix time we received offset for this node */

From 002fcde3da5413cc0d75d0eb313f5f28e2a401ea Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Fri, 8 May 2020 18:10:16 +0200
Subject: [PATCH 147/186] Cluster: refactor ping/data delay handling.

---
 src/cluster.c | 24 +++++++++++++-----------
 1 file changed, 13 insertions(+), 11 deletions(-)

diff --git a/src/cluster.c b/src/cluster.c
index 78524bd34..50c79f4f6 100644
--- a/src/cluster.c
+++ b/src/cluster.c
@@ -3564,15 +3564,17 @@ void clusterCron(void) {
         /* If we are not receiving any data for more than half the cluster
          * timeout, reconnect the link: maybe there is a connection
          * issue even if the node is alive. */
+        mstime_t ping_delay = now - node->ping_sent;
+        mstime_t data_delay = now - node->data_received;
         if (node->link && /* is connected */
             now - node->link->ctime >
             server.cluster_node_timeout && /* was not already reconnected */
             node->ping_sent && /* we already sent a ping */
             node->pong_received < node->ping_sent && /* still waiting pong */
             /* and we are waiting for the pong more than timeout/2 */
-            now - node->ping_sent > server.cluster_node_timeout/2 &&
+            ping_delay > server.cluster_node_timeout/2 &&
             /* and in such interval we are not seeing any traffic at all. */
-            now - node->data_received > server.cluster_node_timeout/2)
+            data_delay > server.cluster_node_timeout/2)
         {
             /* Disconnect the link, it will be reconnected automatically. */
             freeClusterLink(node->link);
@@ -3604,18 +3606,18 @@ void clusterCron(void) {
         /* Check only if we have an active ping for this instance. */
         if (node->ping_sent == 0) continue;
 
-        /* Compute the delay of the PONG. Note that if we already received
-         * the PONG, then node->ping_sent is zero, so can't reach this
-         * code at all. */
-        mstime_t delay = now - node->ping_sent;
-
-        /* We consider every incoming data as proof of liveness, since
+        /* Check if this node looks unreachable.
+         * Note that if we already received the PONG, then node->ping_sent
+         * is zero, so can't reach this code at all, so we don't risk of
+         * checking for a PONG delay if we didn't sent the PING.
+         *
+         * We also consider every incoming data as proof of liveness, since
          * our cluster bus link is also used for data: under heavy data
          * load pong delays are possible. */
-        mstime_t data_delay = now - node->data_received;
-        if (data_delay < delay) delay = data_delay;
+        mstime_t node_delay = (ping_delay < data_delay) ? ping_delay :
+                                                          data_delay;
 
-        if (delay > server.cluster_node_timeout) {
+        if (node_delay > server.cluster_node_timeout) {
             /* Timeout reached. Set the node as possibly failing if it is
              * not already in this state. */
             if (!(node->flags & (CLUSTER_NODE_PFAIL|CLUSTER_NODE_FAIL))) {

From 1276058ea8dfb840db77538e0b35197643d5d680 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Sat, 9 May 2020 11:12:51 +0200
Subject: [PATCH 148/186] Cluster: clarify we always resolve the sender.

---
 src/cluster.c | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/cluster.c b/src/cluster.c
index 50c79f4f6..a2fab323a 100644
--- a/src/cluster.c
+++ b/src/cluster.c
@@ -1741,7 +1741,9 @@ int clusterProcessPacket(clusterLink *link) {
         if (totlen != explen) return 1;
     }
 
-    /* Check if the sender is a known node. */
+    /* Check if the sender is a known node. Note that for incoming connections
+     * we don't store link->node information, but resolve the node by the
+     * ID in the header each time in the current implementation. */
     sender = clusterLookupNode(hdr->sender);
 
     /* Update the last time we saw any data from this node. We

From a23cdbb94b25fda7f49c421c94c85830d0a526c6 Mon Sep 17 00:00:00 2001
From: ShooterIT <wangyuancode@163.com>
Date: Tue, 5 May 2020 23:09:45 +0800
Subject: [PATCH 149/186] Redis Benchmark: Fix coredump because of double free

---
 src/redis-benchmark.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/redis-benchmark.c b/src/redis-benchmark.c
index 77daf981c..34295147d 100644
--- a/src/redis-benchmark.c
+++ b/src/redis-benchmark.c
@@ -279,7 +279,7 @@ static redisConfig *getRedisConfig(const char *ip, int port,
     for (; i < 2; i++) {
         int res = redisGetReply(c, &r);
         if (reply) freeReplyObject(reply);
-        reply = ((redisReply *) r);
+        reply = res == REDIS_OK ? ((redisReply *) r) : NULL;
         if (res != REDIS_OK || !r) goto fail;
         if (reply->type == REDIS_REPLY_ERROR) {
             fprintf(stderr, "ERROR: %s\n", reply->str);

From 3d6f990104ba6f5d6f799b3e43156295ff384487 Mon Sep 17 00:00:00 2001
From: John <john@csquare.ca>
Date: Mon, 11 May 2020 09:22:27 +0000
Subject: [PATCH 150/186] Make multimaster tests more reliable

Former-commit-id: 3122912920973cb433d625a09b183c3f538e2523
---
 tests/integration/replication-multimaster.tcl | 23 +++++++++++--------
 1 file changed, 14 insertions(+), 9 deletions(-)

diff --git a/tests/integration/replication-multimaster.tcl b/tests/integration/replication-multimaster.tcl
index e5e77fdad..63ff51e60 100644
--- a/tests/integration/replication-multimaster.tcl
+++ b/tests/integration/replication-multimaster.tcl
@@ -47,10 +47,13 @@ start_server {overrides {hz 500 active-replica yes multi-master yes}} {
         after 500
         $R(2) incr testkey
         after 500
-        assert_equal 3 [$R(0) get testkey]
-        assert_equal 3 [$R(1) get testkey]
-        assert_equal 3 [$R(2) get testkey]
-        assert_equal 3 [$R(3) get testkey]
+        for {set n 0} {$n < 4} {incr n} {
+            wait_for_condition 50 1000 {
+                [$R($n) get testkey] == 3
+            } else {
+                fail "node $n did not replicate"
+            }
+        }
     }
 
     test "$topology transaction replicates only once" {
@@ -60,11 +63,13 @@ start_server {overrides {hz 500 active-replica yes multi-master yes}} {
             $R(0) incr testkey
             $R(0) incr testkey
             $R(0) exec
-            after 1
-            assert_equal 3 [$R(0) get testkey] "node 0"
-            assert_equal 3 [$R(1) get testkey] "node 1"
-            assert_equal 3 [$R(2) get testkey] "node 2"
-            assert_equal 3 [$R(3) get testkey] "node 3"
+	    for {set n 0} {$n < 4} {incr n} {
+	        wait_for_condition 50 1000 {
+                    [$R($n) get testkey] == 3
+                } else {
+                    fail "node $n failed to replicate"
+                }
+            }
         }
     }
 }

From 181fadb708ff70f691e5aa1ebe2d6234798b215e Mon Sep 17 00:00:00 2001
From: John <john@csquare.ca>
Date: Mon, 11 May 2020 09:37:42 +0000
Subject: [PATCH 151/186] more reliability fixes for multimaster

Former-commit-id: 3543a3c763de91a4d76bca89659fec9bf6b7a1c8
---
 tests/integration/replication-multimaster.tcl | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/tests/integration/replication-multimaster.tcl b/tests/integration/replication-multimaster.tcl
index 63ff51e60..ff7cebac5 100644
--- a/tests/integration/replication-multimaster.tcl
+++ b/tests/integration/replication-multimaster.tcl
@@ -36,8 +36,13 @@ start_server {overrides {hz 500 active-replica yes multi-master yes}} {
     test "$topology replicates to all nodes" {
         $R(0) set testkey foo
         after 500
-        assert_equal foo [$R(1) get testkey] "replicates to 1"
-        assert_equal foo [$R(2) get testkey] "replicates to 2"
+        for {set n 0} {$n < 4} {incr n} {
+            wait_for_condition 50 1000 {
+                [$R($n) get testkey] == "foo"
+            } else {
+                fail "Failed to replicate to $n"
+            }
+        }
     }
 
     test "$topology replicates only once" {

From 5c41802d551477e21df2c568bd0c488a86bcbd0b Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Tue, 12 May 2020 08:59:09 +0300
Subject: [PATCH 152/186] fix unstable replication test

this test which has coverage for varoius flows of diskless master was
failing randomly from time to time.

the failure was:
[err]: diskless all replicas drop during rdb pipe in tests/integration/replication.tcl
log message of '*Diskless rdb transfer, last replica dropped, killing fork child*' not found

what seemed to have happened is that the master didn't detect that all
replicas dropped by the time the replication ended, it thought that one
replica is still connected.

now the test takes a few seconds longer but it seems stable.
---
 tests/integration/replication.tcl | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/integration/replication.tcl b/tests/integration/replication.tcl
index f69002b36..22b0bfeaf 100644
--- a/tests/integration/replication.tcl
+++ b/tests/integration/replication.tcl
@@ -513,9 +513,9 @@ start_server {tags {"repl"}} {
     set master_port [srv 0 port]
     set master_pid [srv 0 pid]
     # put enough data in the db that the rdb file will be bigger than the socket buffers
-    # and since we'll have key-load-delay of 100, 10000 keys will take at least 1 second
+    # and since we'll have key-load-delay of 100, 20000 keys will take at least 2 seconds
     # we also need the replica to process requests during transfer (which it does only once in 2mb)
-    $master debug populate 10000 test 10000
+    $master debug populate 20000 test 10000
     $master config set rdbcompression no
     # If running on Linux, we also measure utime/stime to detect possible I/O handling issues
     set os [catch {exec unamee}]

From e3b5648df083d12c4f2b635deb827732db02607d Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Thu, 14 May 2020 10:41:28 +0200
Subject: [PATCH 153/186] Tracking: send eviction messages when evicting
 entries.

A fix for #7249.
---
 src/server.c   |  6 ++++++
 src/tracking.c | 35 +++++++++++++++++++++++------------
 2 files changed, 29 insertions(+), 12 deletions(-)

diff --git a/src/server.c b/src/server.c
index e2b4b6f3d..847a6a95a 100644
--- a/src/server.c
+++ b/src/server.c
@@ -2053,6 +2053,12 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) {
     /* Stop the I/O threads if we don't have enough pending work. */
     stopThreadedIOIfNeeded();
 
+    /* Resize tracking keys table if needed. This is also done at every
+     * command execution, but we want to be sure that if the last command
+     * executed changes the value via CONFIG SET, the server will perform
+     * the operation even if completely idle. */
+    if (server.tracking_clients) trackingLimitUsedSlots();
+
     /* Start a scheduled BGSAVE if the corresponding flag is set. This is
      * useful when we are forced to postpone a BGSAVE because an AOF
      * rewrite is in progress.
diff --git a/src/tracking.c b/src/tracking.c
index a995817e2..cfde26fc9 100644
--- a/src/tracking.c
+++ b/src/tracking.c
@@ -279,15 +279,22 @@ void trackingRememberKeyToBroadcast(client *c, char *keyname, size_t keylen) {
  *
  * Note that 'c' may be NULL in case the operation was performed outside the
  * context of a client modifying the database (for instance when we delete a
- * key because of expire). */
-void trackingInvalidateKey(client *c, robj *keyobj) {
+ * key because of expire).
+ *
+ * The last argument 'bcast' tells the function if it should also schedule
+ * the key for broadcasting to clients in BCAST mode. This is the case when
+ * the function is called from the Redis core once a key is modified, however
+ * we also call the function in order to evict keys in the key table in case
+ * of memory pressure: in that case the key didn't really change, so we want
+ * just to notify the clients that are in the table for this key, that would
+ * otherwise miss the fact we are no longer tracking the key for them. */
+void trackingInvalidateKeyRaw(client *c, char *key, size_t keylen, int bcast) {
     if (TrackingTable == NULL) return;
-    sds sdskey = keyobj->ptr;
 
-    if (raxSize(PrefixTable) > 0)
-        trackingRememberKeyToBroadcast(c,sdskey,sdslen(sdskey));
+    if (bcast && raxSize(PrefixTable) > 0)
+        trackingRememberKeyToBroadcast(c,key,keylen);
 
-    rax *ids = raxFind(TrackingTable,(unsigned char*)sdskey,sdslen(sdskey));
+    rax *ids = raxFind(TrackingTable,(unsigned char*)key,keylen);
     if (ids == raxNotFound) return;
 
     raxIterator ri;
@@ -317,7 +324,7 @@ void trackingInvalidateKey(client *c, robj *keyobj) {
             continue;
         }
 
-        sendTrackingMessage(target,sdskey,sdslen(sdskey),0);
+        sendTrackingMessage(target,key,keylen,0);
     }
     raxStop(&ri);
 
@@ -325,7 +332,13 @@ void trackingInvalidateKey(client *c, robj *keyobj) {
      * again if more keys will be modified in this caching slot. */
     TrackingTableTotalItems -= raxSize(ids);
     raxFree(ids);
-    raxRemove(TrackingTable,(unsigned char*)sdskey,sdslen(sdskey),NULL);
+    raxRemove(TrackingTable,(unsigned char*)key,keylen,NULL);
+}
+
+/* Wrapper (the one actually called across the core) to pass the key
+ * as object. */
+void trackingInvalidateKey(client *c, robj *keyobj) {
+    trackingInvalidateKeyRaw(c,keyobj->ptr,sdslen(keyobj->ptr),1);
 }
 
 /* This function is called when one or all the Redis databases are flushed
@@ -392,10 +405,8 @@ void trackingLimitUsedSlots(void) {
         effort--;
         raxSeek(&ri,"^",NULL,0);
         raxRandomWalk(&ri,0);
-        rax *ids = ri.data;
-        TrackingTableTotalItems -= raxSize(ids);
-        raxFree(ids);
-        raxRemove(TrackingTable,ri.key,ri.key_len,NULL);
+        if (raxEOF(&ri)) break;
+        trackingInvalidateKeyRaw(NULL,(char*)ri.key,ri.key_len,0);
         if (raxSize(TrackingTable) <= max_keys) {
             timeout_counter = 0;
             raxStop(&ri);

From 693629585b7a2c2f13890fe7001662545436b9a3 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Thu, 14 May 2020 11:17:47 +0200
Subject: [PATCH 154/186] rax.c updated from upstream antirez/rax.

---
 src/rax.c | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/rax.c b/src/rax.c
index a560dde02..7dcf04582 100644
--- a/src/rax.c
+++ b/src/rax.c
@@ -1,6 +1,8 @@
 /* Rax -- A radix tree implementation.
  *
- * Copyright (c) 2017-2018, Salvatore Sanfilippo <antirez at gmail dot com>
+ * Version 1.2 -- 7 February 2019
+ *
+ * Copyright (c) 2017-2019, Salvatore Sanfilippo <antirez at gmail dot com>
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -1737,7 +1739,7 @@ int raxRandomWalk(raxIterator *it, size_t steps) {
     }
 
     if (steps == 0) {
-        size_t fle = floor(log(it->rt->numele));
+        size_t fle = 1+floor(log(it->rt->numele));
         fle *= 2;
         steps = 1 + rand() % fle;
     }

From f7f219a137fcd04ebd79c06fbe8c71e89b638635 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Thu, 14 May 2020 11:27:31 +0200
Subject: [PATCH 155/186] Regression test for #7249.

---
 tests/unit/tracking.tcl | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/tests/unit/tracking.tcl b/tests/unit/tracking.tcl
index 43bb5f864..0332fa726 100644
--- a/tests/unit/tracking.tcl
+++ b/tests/unit/tracking.tcl
@@ -107,5 +107,27 @@ start_server {tags {"tracking"}} {
         assert {$keys eq {mykey}}
     }
 
+    test {Tracking gets notification on tracking table key eviction} {
+        r CLIENT TRACKING off
+        r CLIENT TRACKING on REDIRECT $redir NOLOOP
+        r MSET key1 1 key2 2
+        # Let the server track the two keys for us
+        r MGET key1 key2
+        # Force the eviction of all the keys but one:
+        r config set tracking-table-max-keys 1
+        # Note that we may have other keys in the table for this client,
+        # since we disabled/enabled tracking multiple time with the same
+        # ID, and tracking does not do ID cleanups for performance reasons.
+        # So we check that eventually we'll receive one or the other key,
+        # otherwise the test will die for timeout.
+        while 1 {
+            set keys [lindex [$rd1 read] 2]
+            if {$keys eq {key1} || $keys eq {key2}} break
+        }
+        # We should receive an expire notification for one of
+        # the two keys (only one must remain)
+        assert {$keys eq {key1} || $keys eq {key2}}
+    }
+
     $rd1 close
 }

From 9da134cd88996f5d072451d56b0201b423d984d1 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Sun, 10 May 2020 19:13:47 +0300
Subject: [PATCH 156/186] fix redis 6.0 not freeing closed connections during
 loading.

This bug was introduced by a recent change in which readQueryFromClient
is using freeClientAsync, and despite the fact that now
freeClientsInAsyncFreeQueue is in beforeSleep, that's not enough since
it's not called during loading in processEventsWhileBlocked.
furthermore, afterSleep was called in that case but beforeSleep wasn't.

This bug also caused slowness sine the level-triggered mode of epoll
kept signaling these connections as readable causing us to keep doing
connRead again and again for ll of these, which keep accumulating.

now both before and after sleep are called, but not all of their actions
are performed during loading, some are only reserved for the main loop.

fixes issue #7215
---
 src/ae.c                  |   5 +-
 src/server.c              | 124 ++++++++++++++++++++++----------------
 tests/integration/rdb.tcl |  52 ++++++++++++++++
 3 files changed, 128 insertions(+), 53 deletions(-)

diff --git a/src/ae.c b/src/ae.c
index 1bf6cbfbf..7aa204250 100644
--- a/src/ae.c
+++ b/src/ae.c
@@ -428,6 +428,9 @@ int aeProcessEvents(aeEventLoop *eventLoop, int flags)
             tvp = &tv;
         }
 
+        if (eventLoop->beforesleep != NULL)
+            eventLoop->beforesleep(eventLoop);
+
         /* Call the multiplexing API, will return only on timeout or when
          * some event fires. */
         numevents = aeApiPoll(eventLoop, tvp);
@@ -522,8 +525,6 @@ int aeWait(int fd, int mask, long long milliseconds) {
 void aeMain(aeEventLoop *eventLoop) {
     eventLoop->stop = 0;
     while (!eventLoop->stop) {
-        if (eventLoop->beforesleep != NULL)
-            eventLoop->beforesleep(eventLoop);
         aeProcessEvents(eventLoop, AE_ALL_EVENTS|AE_CALL_AFTER_SLEEP);
     }
 }
diff --git a/src/server.c b/src/server.c
index 847a6a95a..d272a6fe0 100644
--- a/src/server.c
+++ b/src/server.c
@@ -2087,14 +2087,26 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) {
     return 1000/server.hz;
 }
 
+extern int ProcessingEventsWhileBlocked;
+
 /* This function gets called every time Redis is entering the
  * main loop of the event driven library, that is, before to sleep
- * for ready file descriptors. */
+ * for ready file descriptors.
+ * Note: This function is (currently) called from two functions:
+ * 1. aeMain - The main server loop
+ * 2. processEventsWhileBlocked - Process clients during RDB/AOF load
+ * If it was called from processEventsWhileBlocked we don't want
+ * to perform all actions (For example, we don't want to expire
+ * keys), but we do need to perform some actions.
+ * The most important is freeClientsInAsyncFreeQueue but we also
+ * call some other low-risk functions. */
 void beforeSleep(struct aeEventLoop *eventLoop) {
     UNUSED(eventLoop);
 
-    /* Handle precise timeouts of blocked clients. */
-    handleBlockedClientsTimeout();
+    if (!ProcessingEventsWhileBlocked) {
+        /* Handle precise timeouts of blocked clients. */
+        handleBlockedClientsTimeout();
+    }
 
     /* We should handle pending reads clients ASAP after event loop. */
     handleClientsWithPendingReadsUsingThreads();
@@ -2105,65 +2117,69 @@ void beforeSleep(struct aeEventLoop *eventLoop) {
     /* If tls still has pending unread data don't sleep at all. */
     aeSetDontWait(server.el, tlsHasPendingData());
 
-    /* Call the Redis Cluster before sleep function. Note that this function
-     * may change the state of Redis Cluster (from ok to fail or vice versa),
-     * so it's a good idea to call it before serving the unblocked clients
-     * later in this function. */
-    if (server.cluster_enabled) clusterBeforeSleep();
+    if (!ProcessingEventsWhileBlocked) {
+        /* Call the Redis Cluster before sleep function. Note that this function
+         * may change the state of Redis Cluster (from ok to fail or vice versa),
+         * so it's a good idea to call it before serving the unblocked clients
+         * later in this function. */
+        if (server.cluster_enabled) clusterBeforeSleep();
 
-    /* Run a fast expire cycle (the called function will return
-     * ASAP if a fast cycle is not needed). */
-    if (server.active_expire_enabled && server.masterhost == NULL)
-        activeExpireCycle(ACTIVE_EXPIRE_CYCLE_FAST);
+        /* Run a fast expire cycle (the called function will return
+         * ASAP if a fast cycle is not needed). */
+        if (server.active_expire_enabled && server.masterhost == NULL)
+            activeExpireCycle(ACTIVE_EXPIRE_CYCLE_FAST);
 
-    /* Unblock all the clients blocked for synchronous replication
-     * in WAIT. */
-    if (listLength(server.clients_waiting_acks))
-        processClientsWaitingReplicas();
+        /* Unblock all the clients blocked for synchronous replication
+         * in WAIT. */
+        if (listLength(server.clients_waiting_acks))
+            processClientsWaitingReplicas();
 
-    /* Check if there are clients unblocked by modules that implement
-     * blocking commands. */
-    if (moduleCount()) moduleHandleBlockedClients();
+        /* Check if there are clients unblocked by modules that implement
+         * blocking commands. */
+        if (moduleCount()) moduleHandleBlockedClients();
 
-    /* Try to process pending commands for clients that were just unblocked. */
-    if (listLength(server.unblocked_clients))
-        processUnblockedClients();
+        /* Try to process pending commands for clients that were just unblocked. */
+        if (listLength(server.unblocked_clients))
+            processUnblockedClients();
 
-    /* Send all the slaves an ACK request if at least one client blocked
-     * during the previous event loop iteration. Note that we do this after
-     * processUnblockedClients(), so if there are multiple pipelined WAITs
-     * and the just unblocked WAIT gets blocked again, we don't have to wait
-     * a server cron cycle in absence of other event loop events. See #6623. */
-    if (server.get_ack_from_slaves) {
-        robj *argv[3];
+        /* Send all the slaves an ACK request if at least one client blocked
+         * during the previous event loop iteration. Note that we do this after
+         * processUnblockedClients(), so if there are multiple pipelined WAITs
+         * and the just unblocked WAIT gets blocked again, we don't have to wait
+         * a server cron cycle in absence of other event loop events. See #6623. */
+        if (server.get_ack_from_slaves) {
+            robj *argv[3];
 
-        argv[0] = createStringObject("REPLCONF",8);
-        argv[1] = createStringObject("GETACK",6);
-        argv[2] = createStringObject("*",1); /* Not used argument. */
-        replicationFeedSlaves(server.slaves, server.slaveseldb, argv, 3);
-        decrRefCount(argv[0]);
-        decrRefCount(argv[1]);
-        decrRefCount(argv[2]);
-        server.get_ack_from_slaves = 0;
+            argv[0] = createStringObject("REPLCONF",8);
+            argv[1] = createStringObject("GETACK",6);
+            argv[2] = createStringObject("*",1); /* Not used argument. */
+            replicationFeedSlaves(server.slaves, server.slaveseldb, argv, 3);
+            decrRefCount(argv[0]);
+            decrRefCount(argv[1]);
+            decrRefCount(argv[2]);
+            server.get_ack_from_slaves = 0;
+        }
+
+        /* Send the invalidation messages to clients participating to the
+         * client side caching protocol in broadcasting (BCAST) mode. */
+        trackingBroadcastInvalidationMessages();
+
+        /* Write the AOF buffer on disk */
+        flushAppendOnlyFile(0);
     }
 
-    /* Send the invalidation messages to clients participating to the
-     * client side caching protocol in broadcasting (BCAST) mode. */
-    trackingBroadcastInvalidationMessages();
-
-    /* Write the AOF buffer on disk */
-    flushAppendOnlyFile(0);
-
     /* Handle writes with pending output buffers. */
     handleClientsWithPendingWritesUsingThreads();
 
     /* Close clients that need to be closed asynchronous */
     freeClientsInAsyncFreeQueue();
 
-    /* Before we are going to sleep, let the threads access the dataset by
-     * releasing the GIL. Redis main thread will not touch anything at this
-     * time. */
-    if (moduleCount()) moduleReleaseGIL();
+    if (!ProcessingEventsWhileBlocked) {
+        /* Before we are going to sleep, let the threads access the dataset by
+         * releasing the GIL. Redis main thread will not touch anything at this
+         * time. */
+        if (moduleCount()) moduleReleaseGIL();
+    }
 }
 
 /* This function is called immadiately after the event loop multiplexing
@@ -2171,7 +2187,10 @@ void beforeSleep(struct aeEventLoop *eventLoop) {
  * the different events callbacks. */
 void afterSleep(struct aeEventLoop *eventLoop) {
     UNUSED(eventLoop);
-    if (moduleCount()) moduleAcquireGIL();
+
+    if (!ProcessingEventsWhileBlocked) {
+        if (moduleCount()) moduleAcquireGIL();
+    }
 }
 
 /* =========================== Server initialization ======================== */
@@ -2879,6 +2898,11 @@ void initServer(void) {
                 "blocked clients subsystem.");
     }
 
+    /* Register before and after sleep handlers (note this needs to be done
+     * before loading persistence since it is used by processEventsWhileBlocked. */
+    aeSetBeforeSleepProc(server.el,beforeSleep);
+    aeSetAfterSleepProc(server.el,afterSleep);
+
     /* Open the AOF file if needed. */
     if (server.aof_state == AOF_ON) {
         server.aof_fd = open(server.aof_filename,
@@ -5135,8 +5159,6 @@ int main(int argc, char **argv) {
     }
 
     redisSetCpuAffinity(server.server_cpulist);
-    aeSetBeforeSleepProc(server.el,beforeSleep);
-    aeSetAfterSleepProc(server.el,afterSleep);
     aeMain(server.el);
     aeDeleteEventLoop(server.el);
     return 0;
diff --git a/tests/integration/rdb.tcl b/tests/integration/rdb.tcl
index b364291ee..123e9c8b6 100644
--- a/tests/integration/rdb.tcl
+++ b/tests/integration/rdb.tcl
@@ -128,4 +128,56 @@ start_server {} {
         # make sure the server is still writable
         r set x xx
     }
+}
+
+test {client freed during loading} {
+    start_server [list overrides [list key-load-delay 10 rdbcompression no]] {
+        # create a big rdb that will take long to load. it is important
+        # for keys to be big since the server processes events only once in 2mb.
+        # 100mb of rdb, 100k keys will load in more than 1 second
+        r debug populate 100000 key 1000
+
+        catch {
+            r debug restart
+        }
+
+        set stdout [srv 0 stdout]
+        while 1 {
+            # check that the new server actually started and is ready for connections
+            if {[exec grep -i "Server initialized" | wc -l < $stdout] > 1} {
+                break
+            }
+            after 10
+        }
+        # make sure it's still loading
+        assert_equal [s loading] 1
+
+        # connect and disconnect 10 clients
+        set clients {}
+        for {set j 0} {$j < 10} {incr j} {
+            lappend clients [redis_deferring_client]
+        }
+        foreach rd $clients {
+            $rd debug log bla
+        }
+        foreach rd $clients {
+            $rd read
+        }
+        foreach rd $clients {
+            $rd close
+        }
+
+        # make sure the server freed the clients
+        wait_for_condition 100 100 {
+            [s connected_clients] < 3
+        } else {
+            fail "clients didn't disconnect"
+        }
+
+        # make sure it's still loading
+        assert_equal [s loading] 1
+
+        # no need to keep waiting for loading to complete
+        exec kill [srv 0 pid]
+    }
 }
\ No newline at end of file

From 8bf660af90d1a5e33653f18111769f89669ecefc Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Tue, 12 May 2020 13:07:44 +0200
Subject: [PATCH 157/186] Some rework of #7234.

---
 src/ae.c         |   7 ++-
 src/ae.h         |   9 ++--
 src/networking.c |   5 ++-
 src/server.c     | 111 +++++++++++++++++++++++++----------------------
 4 files changed, 72 insertions(+), 60 deletions(-)

diff --git a/src/ae.c b/src/ae.c
index 7aa204250..379cfd1e6 100644
--- a/src/ae.c
+++ b/src/ae.c
@@ -370,6 +370,7 @@ static int processTimeEvents(aeEventLoop *eventLoop) {
  * if flags has AE_DONT_WAIT set the function returns ASAP until all
  * the events that's possible to process without to wait are processed.
  * if flags has AE_CALL_AFTER_SLEEP set, the aftersleep callback is called.
+ * if flags has AE_CALL_BEFORE_SLEEP set, the beforesleep callback is called.
  *
  * The function returns the number of events processed. */
 int aeProcessEvents(aeEventLoop *eventLoop, int flags)
@@ -428,7 +429,7 @@ int aeProcessEvents(aeEventLoop *eventLoop, int flags)
             tvp = &tv;
         }
 
-        if (eventLoop->beforesleep != NULL)
+        if (eventLoop->beforesleep != NULL && flags & AE_CALL_BEFORE_SLEEP)
             eventLoop->beforesleep(eventLoop);
 
         /* Call the multiplexing API, will return only on timeout or when
@@ -525,7 +526,9 @@ int aeWait(int fd, int mask, long long milliseconds) {
 void aeMain(aeEventLoop *eventLoop) {
     eventLoop->stop = 0;
     while (!eventLoop->stop) {
-        aeProcessEvents(eventLoop, AE_ALL_EVENTS|AE_CALL_AFTER_SLEEP);
+        aeProcessEvents(eventLoop, AE_ALL_EVENTS|
+                                   AE_CALL_BEFORE_SLEEP|
+                                   AE_CALL_AFTER_SLEEP);
     }
 }
 
diff --git a/src/ae.h b/src/ae.h
index 9acd72434..63b306615 100644
--- a/src/ae.h
+++ b/src/ae.h
@@ -47,11 +47,12 @@
                            things to disk before sending replies, and want
                            to do that in a group fashion. */
 
-#define AE_FILE_EVENTS 1
-#define AE_TIME_EVENTS 2
+#define AE_FILE_EVENTS (1<<0)
+#define AE_TIME_EVENTS (1<<1)
 #define AE_ALL_EVENTS (AE_FILE_EVENTS|AE_TIME_EVENTS)
-#define AE_DONT_WAIT 4
-#define AE_CALL_AFTER_SLEEP 8
+#define AE_DONT_WAIT (1<<2)
+#define AE_CALL_BEFORE_SLEEP (1<<3)
+#define AE_CALL_AFTER_SLEEP (1<<4)
 
 #define AE_NOMORE -1
 #define AE_DELETED_EVENT_ID -1
diff --git a/src/networking.c b/src/networking.c
index 75c0c16b1..ce5b0ae38 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -2863,8 +2863,9 @@ int processEventsWhileBlocked(void) {
     ProcessingEventsWhileBlocked = 1;
     while (iterations--) {
         int events = 0;
-        events += aeProcessEvents(server.el, AE_FILE_EVENTS|AE_DONT_WAIT);
-        events += handleClientsWithPendingWrites();
+        events += aeProcessEvents(server.el,
+            AE_FILE_EVENTS|AE_DONT_WAIT|
+            AE_CALL_BEFORE_SLEEP|AE_CALL_AFTER_SLEEP);
         if (!events) break;
         count += events;
     }
diff --git a/src/server.c b/src/server.c
index d272a6fe0..73a4a9f55 100644
--- a/src/server.c
+++ b/src/server.c
@@ -2092,22 +2092,33 @@ extern int ProcessingEventsWhileBlocked;
 /* This function gets called every time Redis is entering the
  * main loop of the event driven library, that is, before to sleep
  * for ready file descriptors.
+ *
  * Note: This function is (currently) called from two functions:
  * 1. aeMain - The main server loop
  * 2. processEventsWhileBlocked - Process clients during RDB/AOF load
+ *
  * If it was called from processEventsWhileBlocked we don't want
  * to perform all actions (For example, we don't want to expire
  * keys), but we do need to perform some actions.
+ *
  * The most important is freeClientsInAsyncFreeQueue but we also
  * call some other low-risk functions. */
 void beforeSleep(struct aeEventLoop *eventLoop) {
     UNUSED(eventLoop);
 
-    if (!ProcessingEventsWhileBlocked) {
-        /* Handle precise timeouts of blocked clients. */
-        handleBlockedClientsTimeout();
+    /* Just call a subset of vital functions in case we are re-entering
+     * the event loop from processEventsWhileBlocked(). */
+    if (ProcessingEventsWhileBlocked) {
+        handleClientsWithPendingReadsUsingThreads();
+        tlsProcessPendingData();
+        handleClientsWithPendingWrites();
+        freeClientsInAsyncFreeQueue();
+        return;
     }
 
+    /* Handle precise timeouts of blocked clients. */
+    handleBlockedClientsTimeout();
+
     /* We should handle pending reads clients ASAP after event loop. */
     handleClientsWithPendingReadsUsingThreads();
 
@@ -2117,69 +2128,65 @@ void beforeSleep(struct aeEventLoop *eventLoop) {
     /* If tls still has pending unread data don't sleep at all. */
     aeSetDontWait(server.el, tlsHasPendingData());
 
-    if (!ProcessingEventsWhileBlocked) {
-        /* Call the Redis Cluster before sleep function. Note that this function
-         * may change the state of Redis Cluster (from ok to fail or vice versa),
-         * so it's a good idea to call it before serving the unblocked clients
-         * later in this function. */
-        if (server.cluster_enabled) clusterBeforeSleep();
+    /* Call the Redis Cluster before sleep function. Note that this function
+     * may change the state of Redis Cluster (from ok to fail or vice versa),
+     * so it's a good idea to call it before serving the unblocked clients
+     * later in this function. */
+    if (server.cluster_enabled) clusterBeforeSleep();
 
-        /* Run a fast expire cycle (the called function will return
-         * ASAP if a fast cycle is not needed). */
-        if (server.active_expire_enabled && server.masterhost == NULL)
-            activeExpireCycle(ACTIVE_EXPIRE_CYCLE_FAST);
+    /* Run a fast expire cycle (the called function will return
+     * ASAP if a fast cycle is not needed). */
+    if (server.active_expire_enabled && server.masterhost == NULL)
+        activeExpireCycle(ACTIVE_EXPIRE_CYCLE_FAST);
 
-        /* Unblock all the clients blocked for synchronous replication
-         * in WAIT. */
-        if (listLength(server.clients_waiting_acks))
-            processClientsWaitingReplicas();
+    /* Unblock all the clients blocked for synchronous replication
+     * in WAIT. */
+    if (listLength(server.clients_waiting_acks))
+        processClientsWaitingReplicas();
 
-        /* Check if there are clients unblocked by modules that implement
-         * blocking commands. */
-        if (moduleCount()) moduleHandleBlockedClients();
+    /* Check if there are clients unblocked by modules that implement
+     * blocking commands. */
+    if (moduleCount()) moduleHandleBlockedClients();
 
-        /* Try to process pending commands for clients that were just unblocked. */
-        if (listLength(server.unblocked_clients))
-            processUnblockedClients();
+    /* Try to process pending commands for clients that were just unblocked. */
+    if (listLength(server.unblocked_clients))
+        processUnblockedClients();
 
-        /* Send all the slaves an ACK request if at least one client blocked
-         * during the previous event loop iteration. Note that we do this after
-         * processUnblockedClients(), so if there are multiple pipelined WAITs
-         * and the just unblocked WAIT gets blocked again, we don't have to wait
-         * a server cron cycle in absence of other event loop events. See #6623. */
-        if (server.get_ack_from_slaves) {
-            robj *argv[3];
+    /* Send all the slaves an ACK request if at least one client blocked
+     * during the previous event loop iteration. Note that we do this after
+     * processUnblockedClients(), so if there are multiple pipelined WAITs
+     * and the just unblocked WAIT gets blocked again, we don't have to wait
+     * a server cron cycle in absence of other event loop events. See #6623. */
+    if (server.get_ack_from_slaves) {
+        robj *argv[3];
 
-            argv[0] = createStringObject("REPLCONF",8);
-            argv[1] = createStringObject("GETACK",6);
-            argv[2] = createStringObject("*",1); /* Not used argument. */
-            replicationFeedSlaves(server.slaves, server.slaveseldb, argv, 3);
-            decrRefCount(argv[0]);
-            decrRefCount(argv[1]);
-            decrRefCount(argv[2]);
-            server.get_ack_from_slaves = 0;
-        }
-
-        /* Send the invalidation messages to clients participating to the
-         * client side caching protocol in broadcasting (BCAST) mode. */
-        trackingBroadcastInvalidationMessages();
-
-        /* Write the AOF buffer on disk */
-        flushAppendOnlyFile(0);
+        argv[0] = createStringObject("REPLCONF",8);
+        argv[1] = createStringObject("GETACK",6);
+        argv[2] = createStringObject("*",1); /* Not used argument. */
+        replicationFeedSlaves(server.slaves, server.slaveseldb, argv, 3);
+        decrRefCount(argv[0]);
+        decrRefCount(argv[1]);
+        decrRefCount(argv[2]);
+        server.get_ack_from_slaves = 0;
     }
 
+    /* Send the invalidation messages to clients participating to the
+     * client side caching protocol in broadcasting (BCAST) mode. */
+    trackingBroadcastInvalidationMessages();
+
+    /* Write the AOF buffer on disk */
+    flushAppendOnlyFile(0);
+
     /* Handle writes with pending output buffers. */
     handleClientsWithPendingWritesUsingThreads();
 
     /* Close clients that need to be closed asynchronous */
     freeClientsInAsyncFreeQueue();
 
-    if (!ProcessingEventsWhileBlocked) {
-        /* Before we are going to sleep, let the threads access the dataset by
-         * releasing the GIL. Redis main thread will not touch anything at this
-         * time. */
-        if (moduleCount()) moduleReleaseGIL();
-    }
+    /* Before we are going to sleep, let the threads access the dataset by
+     * releasing the GIL. Redis main thread will not touch anything at this
+     * time. */
+    if (moduleCount()) moduleReleaseGIL();
 }
 
 /* This function is called immadiately after the event loop multiplexing

From 74249be4a24a8da90f87e9898fb456a49141c57c Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Thu, 14 May 2020 10:02:57 +0200
Subject: [PATCH 158/186] Track events processed while blocked globally.

Related to #7234.
---
 src/connection.h |  2 +-
 src/networking.c | 19 ++++++++++++-------
 src/server.c     | 16 +++++++++++-----
 src/server.h     |  5 +++--
 src/tls.c        |  7 +++++--
 5 files changed, 32 insertions(+), 17 deletions(-)

diff --git a/src/connection.h b/src/connection.h
index db09dfd83..0fd6c5f24 100644
--- a/src/connection.h
+++ b/src/connection.h
@@ -222,6 +222,6 @@ const char *connGetInfo(connection *conn, char *buf, size_t buf_len);
 
 /* Helpers for tls special considerations */
 int tlsHasPendingData();
-void tlsProcessPendingData();
+int tlsProcessPendingData();
 
 #endif  /* __REDIS_CONNECTION_H */
diff --git a/src/networking.c b/src/networking.c
index ce5b0ae38..671e374f4 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -1257,7 +1257,10 @@ void freeClientAsync(client *c) {
     pthread_mutex_unlock(&async_free_queue_mutex);
 }
 
-void freeClientsInAsyncFreeQueue(void) {
+/* Free the clietns marked as CLOSE_ASAP, return the number of clients
+ * freed. */
+int freeClientsInAsyncFreeQueue(void) {
+    int freed = listLength(server.clients_to_close);
     while (listLength(server.clients_to_close)) {
         listNode *ln = listFirst(server.clients_to_close);
         client *c = listNodeValue(ln);
@@ -1266,6 +1269,7 @@ void freeClientsInAsyncFreeQueue(void) {
         freeClient(c);
         listDelNode(server.clients_to_close,ln);
     }
+    return freed;
 }
 
 /* Return a client by ID, or NULL if the client ID is not in the set
@@ -2852,9 +2856,8 @@ int clientsArePaused(void) {
  * write, close sequence needed to serve a client.
  *
  * The function returns the total number of events processed. */
-int processEventsWhileBlocked(void) {
+void processEventsWhileBlocked(void) {
     int iterations = 4; /* See the function top-comment. */
-    int count = 0;
 
     /* Note: when we are processing events while blocked (for instance during
      * busy Lua scripts), we set a global flag. When such flag is set, we
@@ -2862,15 +2865,17 @@ int processEventsWhileBlocked(void) {
      * See https://github.com/antirez/redis/issues/6988 for more info. */
     ProcessingEventsWhileBlocked = 1;
     while (iterations--) {
-        int events = 0;
-        events += aeProcessEvents(server.el,
+        long long startval = server.events_processed_while_blocked;
+        long long ae_events = aeProcessEvents(server.el,
             AE_FILE_EVENTS|AE_DONT_WAIT|
             AE_CALL_BEFORE_SLEEP|AE_CALL_AFTER_SLEEP);
+        /* Note that server.events_processed_while_blocked will also get
+         * incremeted by callbacks called by the event loop handlers. */
+        server.events_processed_while_blocked += ae_events;
+        long long events = server.events_processed_while_blocked - startval;
         if (!events) break;
-        count += events;
     }
     ProcessingEventsWhileBlocked = 0;
-    return count;
 }
 
 /* ==========================================================================
diff --git a/src/server.c b/src/server.c
index 73a4a9f55..5bc4666ee 100644
--- a/src/server.c
+++ b/src/server.c
@@ -2107,12 +2107,17 @@ void beforeSleep(struct aeEventLoop *eventLoop) {
     UNUSED(eventLoop);
 
     /* Just call a subset of vital functions in case we are re-entering
-     * the event loop from processEventsWhileBlocked(). */
+     * the event loop from processEventsWhileBlocked(). Note that in this
+     * case we keep track of the number of events we are processing, since
+     * processEventsWhileBlocked() wants to stop ASAP if there are no longer
+     * events to handle. */
     if (ProcessingEventsWhileBlocked) {
-        handleClientsWithPendingReadsUsingThreads();
-        tlsProcessPendingData();
-        handleClientsWithPendingWrites();
-        freeClientsInAsyncFreeQueue();
+        uint64_t processed = 0;
+        processed += handleClientsWithPendingReadsUsingThreads();
+        processed += tlsProcessPendingData();
+        processed += handleClientsWithPendingWrites();
+        processed += freeClientsInAsyncFreeQueue();
+        server.events_processed_while_blocked += processed;
         return;
     }
 
@@ -2763,6 +2768,7 @@ void initServer(void) {
     server.clients_waiting_acks = listCreate();
     server.get_ack_from_slaves = 0;
     server.clients_paused = 0;
+    server.events_processed_while_blocked = 0;
     server.system_memory_size = zmalloc_get_memory_size();
 
     if (server.tls_port && tlsConfigure(&server.tls_ctx_config) == C_ERR) {
diff --git a/src/server.h b/src/server.h
index 59cf1370e..55ee2d300 100644
--- a/src/server.h
+++ b/src/server.h
@@ -1095,6 +1095,7 @@ struct redisServer {
                                    queries. Will still serve RESP2 queries. */
     int io_threads_num;         /* Number of IO threads to use. */
     int io_threads_do_reads;    /* Read and parse from IO threads? */
+    long long events_processed_while_blocked; /* processEventsWhileBlocked() */
 
     /* RDB / AOF loading information */
     int loading;                /* We are loading data from disk if true */
@@ -1652,7 +1653,7 @@ void rewriteClientCommandVector(client *c, int argc, ...);
 void rewriteClientCommandArgument(client *c, int i, robj *newval);
 void replaceClientCommandVector(client *c, int argc, robj **argv);
 unsigned long getClientOutputBufferMemoryUsage(client *c);
-void freeClientsInAsyncFreeQueue(void);
+int freeClientsInAsyncFreeQueue(void);
 void asyncCloseClientOnOutputBufferLimitReached(client *c);
 int getClientType(client *c);
 int getClientTypeByName(char *name);
@@ -1662,7 +1663,7 @@ void disconnectSlaves(void);
 int listenToPort(int port, int *fds, int *count);
 void pauseClients(mstime_t duration);
 int clientsArePaused(void);
-int processEventsWhileBlocked(void);
+void processEventsWhileBlocked(void);
 int handleClientsWithPendingWrites(void);
 int handleClientsWithPendingWritesUsingThreads(void);
 int handleClientsWithPendingReadsUsingThreads(void);
diff --git a/src/tls.c b/src/tls.c
index c18aafebe..ee85bd302 100644
--- a/src/tls.c
+++ b/src/tls.c
@@ -768,15 +768,17 @@ int tlsHasPendingData() {
     return listLength(pending_list) > 0;
 }
 
-void tlsProcessPendingData() {
+int tlsProcessPendingData() {
     listIter li;
     listNode *ln;
 
+    int processed = listLength(pending_list);
     listRewind(pending_list,&li);
     while((ln = listNext(&li))) {
         tls_connection *conn = listNodeValue(ln);
         tlsHandleEvent(conn, AE_READABLE);
     }
+    return processed;
 }
 
 #else   /* USE_OPENSSL */
@@ -804,7 +806,8 @@ int tlsHasPendingData() {
     return 0;
 }
 
-void tlsProcessPendingData() {
+int tlsProcessPendingData() {
+    return 0;
 }
 
 #endif

From d32cd50f2511e7215288e1541f651d05882fa911 Mon Sep 17 00:00:00 2001
From: John Sully <john@csquare.ca>
Date: Thu, 14 May 2020 22:18:29 -0400
Subject: [PATCH 159/186] Remove debugging logs

Former-commit-id: 407375887429ebd3efd6e21ae58766397d1324d3
---
 src/module.cpp | 2 --
 1 file changed, 2 deletions(-)

diff --git a/src/module.cpp b/src/module.cpp
index 60420686e..47809a03a 100644
--- a/src/module.cpp
+++ b/src/module.cpp
@@ -7765,11 +7765,9 @@ int RM_GetLFU(RedisModuleKey *key, long long *lfu_freq) {
     *lfu_freq = -1;
     if (!key->value)
         return REDISMODULE_ERR;
-    serverLog(LL_WARNING, "MAXMEMORY_POLICY: %X", g_pserver->maxmemory_policy);
     if (g_pserver->maxmemory_policy & MAXMEMORY_FLAG_LFU)
     {
         *lfu_freq = LFUDecrAndReturn(key->value);
-        serverLog(LL_WARNING, "lfu_freq: %lld", lfu_freq);
     }
     return REDISMODULE_OK;
 }

From 80c906bd30117b02f72b4b9251c34bb85e3c4703 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Fri, 15 May 2020 10:19:13 +0200
Subject: [PATCH 160/186] Cache master without checking of deferred close
 flags.

The context is issue #7205: since the introduction of threaded I/O we close
clients asynchronously by default from readQueryFromClient(). So we
should no longer prevent the caching of the master client, to later
PSYNC incrementally, if such flags are set. However we also don't want
the master client to be cached with such flags (would be closed
immediately after being restored). And yet we want a way to understand
if a master was closed because of a protocol error, and in that case
prevent the caching.
---
 src/networking.c  | 10 ++++------
 src/replication.c |  8 ++++++--
 src/server.h      |  1 +
 3 files changed, 11 insertions(+), 8 deletions(-)

diff --git a/src/networking.c b/src/networking.c
index 671e374f4..6de00be09 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -1141,10 +1141,7 @@ void freeClient(client *c) {
      * some unexpected state, by checking its flags. */
     if (server.master && c->flags & CLIENT_MASTER) {
         serverLog(LL_WARNING,"Connection with master lost.");
-        if (!(c->flags & (CLIENT_CLOSE_AFTER_REPLY|
-                          CLIENT_CLOSE_ASAP|
-                          CLIENT_BLOCKED)))
-        {
+        if (!(c->flags & (CLIENT_PROTOCOL_ERROR|CLIENT_BLOCKED))) {
             replicationCacheMaster(c);
             return;
         }
@@ -1558,7 +1555,8 @@ int processInlineBuffer(client *c) {
 }
 
 /* Helper function. Record protocol erro details in server log,
- * and set the client as CLIENT_CLOSE_AFTER_REPLY. */
+ * and set the client as CLIENT_CLOSE_AFTER_REPLY and
+ * CLIENT_PROTOCOL_ERROR. */
 #define PROTO_DUMP_LEN 128
 static void setProtocolError(const char *errstr, client *c) {
     if (server.verbosity <= LL_VERBOSE) {
@@ -1584,7 +1582,7 @@ static void setProtocolError(const char *errstr, client *c) {
             "Protocol error (%s) from client: %s. %s", errstr, client, buf);
         sdsfree(client);
     }
-    c->flags |= CLIENT_CLOSE_AFTER_REPLY;
+    c->flags |= (CLIENT_CLOSE_AFTER_REPLY|CLIENT_PROTOCOL_ERROR);
 }
 
 /* Process the query buffer for client 'c', setting up the client argument
diff --git a/src/replication.c b/src/replication.c
index c59639cd1..f433d6413 100644
--- a/src/replication.c
+++ b/src/replication.c
@@ -2688,14 +2688,18 @@ void replicationCacheMaster(client *c) {
     /* Unlink the client from the server structures. */
     unlinkClient(c);
 
+    /* Clear flags that can create issues once we reconnect the client. */
+    c->flags &= ~(CLIENT_CLOSE_ASAP|CLIENT_CLOSE_AFTER_REPLY);
+
     /* Reset the master client so that's ready to accept new commands:
      * we want to discard te non processed query buffers and non processed
      * offsets, including pending transactions, already populated arguments,
      * pending outputs to the master. */
     sdsclear(server.master->querybuf);
     sdsclear(server.master->pending_querybuf);
-    /* Adjust reploff and read_reploff to the last meaningful offset we executed.
-     * this is the offset the replica will use for future PSYNC. */
+
+    /* Adjust reploff and read_reploff to the last meaningful offset we
+     * executed. This is the offset the replica will use for future PSYNC. */
     server.master->reploff = adjustMeaningfulReplOffset();
     server.master->read_reploff = server.master->reploff;
     if (c->flags & CLIENT_MULTI) discardTransaction(c);
diff --git a/src/server.h b/src/server.h
index 55ee2d300..f835bf5e9 100644
--- a/src/server.h
+++ b/src/server.h
@@ -255,6 +255,7 @@ typedef long long ustime_t; /* microsecond time type. */
 #define CLIENT_TRACKING_NOLOOP (1ULL<<37) /* Don't send invalidation messages
                                              about writes performed by myself.*/
 #define CLIENT_IN_TO_TABLE (1ULL<<38) /* This client is in the timeout table. */
+#define CLIENT_PROTOCOL_ERROR (1ULL<<39) /* Protocol error chatting with it. */
 
 /* Client block type (btype field in client structure)
  * if CLIENT_BLOCKED flag is set. */

From 2435341d7450c7e81e2facc4aebe981e1c900cc4 Mon Sep 17 00:00:00 2001
From: Madelyn Olson <matolson@amazon.com>
Date: Thu, 14 May 2020 08:37:24 -0700
Subject: [PATCH 161/186] Added a refcount on timer events to prevent deletion
 of recursive timer calls

---
 src/ae.c | 10 ++++++++++
 src/ae.h |  2 ++
 2 files changed, 12 insertions(+)

diff --git a/src/ae.c b/src/ae.c
index 379cfd1e6..689a27d16 100644
--- a/src/ae.c
+++ b/src/ae.c
@@ -238,6 +238,7 @@ long long aeCreateTimeEvent(aeEventLoop *eventLoop, long long milliseconds,
     te->clientData = clientData;
     te->prev = NULL;
     te->next = eventLoop->timeEventHead;
+    te->refcount = 0;
     if (te->next)
         te->next->prev = te;
     eventLoop->timeEventHead = te;
@@ -316,6 +317,13 @@ static int processTimeEvents(aeEventLoop *eventLoop) {
         /* Remove events scheduled for deletion. */
         if (te->id == AE_DELETED_EVENT_ID) {
             aeTimeEvent *next = te->next;
+            /* If a reference exists for this timer event,
+             * don't free it. This is currently incremented
+             * for recursive timerProc calls */
+            if (te->refcount) {
+                te = next;
+                continue;
+            }
             if (te->prev)
                 te->prev->next = te->next;
             else
@@ -345,7 +353,9 @@ static int processTimeEvents(aeEventLoop *eventLoop) {
             int retval;
 
             id = te->id;
+            te->refcount++;
             retval = te->timeProc(eventLoop, id, te->clientData);
+            te->refcount--;
             processed++;
             if (retval != AE_NOMORE) {
                 aeAddMillisecondsToNow(retval,&te->when_sec,&te->when_ms);
diff --git a/src/ae.h b/src/ae.h
index 63b306615..d1b7f34bf 100644
--- a/src/ae.h
+++ b/src/ae.h
@@ -86,6 +86,8 @@ typedef struct aeTimeEvent {
     void *clientData;
     struct aeTimeEvent *prev;
     struct aeTimeEvent *next;
+    int refcount; /* refcount to prevent timer events from being
+  		   * freed in recursive time event calls. */
 } aeTimeEvent;
 
 /* A fired event */

From 389697988a78e131e326b1b02005b0deec4a8a43 Mon Sep 17 00:00:00 2001
From: David Carlier <devnexen@gmail.com>
Date: Tue, 12 May 2020 21:19:12 +0100
Subject: [PATCH 162/186] NetBSD build update.

This platform supports CPU affinity (but not OpenBSD).
---
 src/Makefile         | 10 ++++++++++
 src/config.h         |  2 +-
 src/setcpuaffinity.c | 19 +++++++++++++++++++
 3 files changed, 30 insertions(+), 1 deletion(-)

diff --git a/src/Makefile b/src/Makefile
index 55f862cfc..b8c05c32b 100644
--- a/src/Makefile
+++ b/src/Makefile
@@ -128,6 +128,14 @@ else
 ifeq ($(uname_S),DragonFly)
 	# FreeBSD
 	FINAL_LIBS+= -lpthread -lexecinfo
+else
+ifeq ($(uname_S),OpenBSD)
+	# OpenBSD
+	FINAL_LIBS+= -lpthread -lexecinfo
+else
+ifeq ($(uname_S),NetBSD)
+	# NetBSD
+	FINAL_LIBS+= -lpthread -lexecinfo
 else
 	# All the other OSes (notably Linux)
 	FINAL_LDFLAGS+= -rdynamic
@@ -138,6 +146,8 @@ endif
 endif
 endif
 endif
+endif
+endif
 # Include paths to dependencies
 FINAL_CFLAGS+= -I../deps/hiredis -I../deps/linenoise -I../deps/lua/src
 
diff --git a/src/config.h b/src/config.h
index 6025d4e96..0fcc42972 100644
--- a/src/config.h
+++ b/src/config.h
@@ -248,7 +248,7 @@ int pthread_setname_np(const char *name);
 #endif
 
 /* Check if we can use setcpuaffinity(). */
-#if (defined __linux || defined __NetBSD__ || defined __FreeBSD__ || defined __OpenBSD__)
+#if (defined __linux || defined __NetBSD__ || defined __FreeBSD__)
 #define USE_SETCPUAFFINITY
 void setcpuaffinity(const char *cpulist);
 #endif
diff --git a/src/setcpuaffinity.c b/src/setcpuaffinity.c
index dcae81c71..13594113c 100644
--- a/src/setcpuaffinity.c
+++ b/src/setcpuaffinity.c
@@ -36,6 +36,10 @@
 #include <sys/param.h>
 #include <sys/cpuset.h>
 #endif
+#ifdef __NetBSD__
+#include <pthread.h>
+#include <sched.h>
+#endif
 #include "config.h"
 
 #ifdef USE_SETCPUAFFINITY
@@ -71,11 +75,18 @@ void setcpuaffinity(const char *cpulist) {
 #ifdef __FreeBSD__
     cpuset_t cpuset;
 #endif
+#ifdef __NetBSD__
+    cpuset_t *cpuset;
+#endif
 
     if (!cpulist)
         return;
 
+#ifndef __NetBSD__
     CPU_ZERO(&cpuset);
+#else
+    cpuset = cpuset_create();
+#endif
 
     q = cpulist;
     while (p = q, q = next_token(q, ','), p) {
@@ -110,7 +121,11 @@ void setcpuaffinity(const char *cpulist) {
             return;
 
         while (a <= b) {
+#ifndef __NetBSD__
             CPU_SET(a, &cpuset);
+#else
+            cpuset_set(a, cpuset);
+#endif
             a += s;
         }
     }
@@ -124,6 +139,10 @@ void setcpuaffinity(const char *cpulist) {
 #ifdef __FreeBSD__
     cpuset_setaffinity(CPU_LEVEL_WHICH, CPU_WHICH_TID, -1, sizeof(cpuset), &cpuset);
 #endif
+#ifdef __NetBSD__
+    pthread_setaffinity_np(pthread_self(), cpuset_size(cpuset), cpuset);
+    cpuset_destroy(cpuset);
+#endif
 }
 
 #endif /* USE_SETCPUAFFINITY */

From 77ae66930c47b4b3c685bd534a043b86d9df5900 Mon Sep 17 00:00:00 2001
From: Yossi Gottlieb <yossigo@gmail.com>
Date: Sun, 10 May 2020 17:35:27 +0300
Subject: [PATCH 163/186] TLS: Add crypto locks for older OpenSSL support.

This is really required only for older OpenSSL versions.

Also, at the moment Redis does not use OpenSSL from multiple threads so
this will only be useful if modules end up doing that.
---
 src/tls.c | 45 +++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 45 insertions(+)

diff --git a/src/tls.c b/src/tls.c
index ee85bd302..28a74df9a 100644
--- a/src/tls.c
+++ b/src/tls.c
@@ -93,11 +93,56 @@ static int parseProtocolsConfig(const char *str) {
  * served to the reader yet. */
 static list *pending_list = NULL;
 
+/**
+ * OpenSSL global initialization and locking handling callbacks.
+ * Note that this is only required for OpenSSL < 1.1.0.
+ */
+
+#if OPENSSL_VERSION_NUMBER < 0x10100000L
+#define USE_CRYPTO_LOCKS
+#endif
+
+#ifdef USE_CRYPTO_LOCKS
+
+static pthread_mutex_t *openssl_locks;
+
+static void sslLockingCallback(int mode, int lock_id, const char *f, int line) {
+    pthread_mutex_t *mt = openssl_locks + lock_id;
+
+    if (mode & CRYPTO_LOCK) {
+        pthread_mutex_lock(mt);
+    } else {
+        pthread_mutex_unlock(mt);
+    }
+
+    (void)f;
+    (void)line;
+}
+
+static void initCryptoLocks(void) {
+    unsigned i, nlocks;
+    if (CRYPTO_get_locking_callback() != NULL) {
+        /* Someone already set the callback before us. Don't destroy it! */
+        return;
+    }
+    nlocks = CRYPTO_num_locks();
+    openssl_locks = zmalloc(sizeof(*openssl_locks) * nlocks);
+    for (i = 0; i < nlocks; i++) {
+        pthread_mutex_init(openssl_locks + i, NULL);
+    }
+    CRYPTO_set_locking_callback(sslLockingCallback);
+}
+#endif /* USE_CRYPTO_LOCKS */
+
 void tlsInit(void) {
     ERR_load_crypto_strings();
     SSL_load_error_strings();
     SSL_library_init();
 
+#ifdef USE_CRYPTO_LOCKS
+    initCryptoLocks();
+#endif
+
     if (!RAND_poll()) {
         serverLog(LL_WARNING, "OpenSSL: Failed to seed random number generator.");
     }

From 16ba33c05bf94b94ec8217e82aa50a1aeef62573 Mon Sep 17 00:00:00 2001
From: Yossi Gottlieb <yossigo@gmail.com>
Date: Sun, 10 May 2020 17:38:04 +0300
Subject: [PATCH 164/186] TLS: Fix test failures on recent Debian/Ubuntu.

Seems like on some systems choosing specific TLS v1/v1.1 versions no
longer works as expected. Test is reduced for v1.2 now which is still
good enough to test the mechansim, and matters most anyway.
---
 tests/unit/tls.tcl | 20 --------------------
 1 file changed, 20 deletions(-)

diff --git a/tests/unit/tls.tcl b/tests/unit/tls.tcl
index 950f65557..2b04590cd 100644
--- a/tests/unit/tls.tcl
+++ b/tests/unit/tls.tcl
@@ -25,26 +25,6 @@ start_server {tags {"tls"}} {
         }
 
         test {TLS: Verify tls-protocols behaves as expected} {
-            r CONFIG SET tls-protocols TLSv1
-
-            set s [redis [srv 0 host] [srv 0 port] 0 1 {-tls1 0}]
-            catch {$s PING} e
-            assert_match {*I/O error*} $e
-
-            set s [redis [srv 0 host] [srv 0 port] 0 1 {-tls1 1}]
-            catch {$s PING} e
-            assert_match {PONG} $e
-
-            r CONFIG SET tls-protocols TLSv1.1
-
-            set s [redis [srv 0 host] [srv 0 port] 0 1 {-tls1.1 0}]
-            catch {$s PING} e
-            assert_match {*I/O error*} $e
-
-            set s [redis [srv 0 host] [srv 0 port] 0 1 {-tls1.1 1}]
-            catch {$s PING} e
-            assert_match {PONG} $e
-
             r CONFIG SET tls-protocols TLSv1.2
 
             set s [redis [srv 0 host] [srv 0 port] 0 1 {-tls1.2 0}]

From ae306a3df6cf63b31a0814cb5393a9df59947d2e Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Fri, 15 May 2020 22:29:52 +0200
Subject: [PATCH 165/186] Redis 6.0.2.

---
 00-RELEASENOTES | 179 ++++++++++++++++++++++++++++++++++++++++++++++++
 src/version.h   |   2 +-
 2 files changed, 180 insertions(+), 1 deletion(-)

diff --git a/00-RELEASENOTES b/00-RELEASENOTES
index 1d5486ae8..1fa620bd8 100644
--- a/00-RELEASENOTES
+++ b/00-RELEASENOTES
@@ -11,6 +11,185 @@ CRITICAL: There is a critical bug affecting MOST USERS. Upgrade ASAP.
 SECURITY: There are security fixes in the release.
 --------------------------------------------------------------------------------
 
+================================================================================
+Redis 6.0.2     Released Fri May 15 22:24:36 CEST 2020
+================================================================================
+
+Upgrade urgency MODERATE: many not critical bugfixes in different areas.
+                          Critical fix to client side caching when
+                          keys are evicted from the tracking table but
+                          no notifications are sent.
+
+The following are the most serious fix:
+
+* XPENDING should not update consumer's seen-time
+* optimize memory usage of deferred replies - fixed
+* Fix CRC64 initialization outside the Redis server itself.
+* stringmatchlen() should not expect null terminated strings.
+* Cluster nodes availability checks improved when there is
+  high Pub/Sub load on the cluster bus.
+* Redis Benchmark: Fix coredump because of double free
+* Tracking: send eviction messages when evicting entries.
+* rax.c updated from upstream antirez/rax.
+* fix redis 6.0 not freeing closed connections during loading.
+
+New features:
+
+* Support setcpuaffinity on linux/bsd
+* Client Side Caching: Add Tracking Prefix Number Stats in Server Info
+* Add --user argument to redis-benchmark.c (ACL)
+
+Full list of commits:
+
+Yossi Gottlieb in commit 16ba33c05:
+ TLS: Fix test failures on recent Debian/Ubuntu.
+ 1 file changed, 20 deletions(-)
+
+Yossi Gottlieb in commit 77ae66930:
+ TLS: Add crypto locks for older OpenSSL support.
+ 1 file changed, 45 insertions(+)
+
+David Carlier in commit 389697988:
+ NetBSD build update.
+ 3 files changed, 30 insertions(+), 1 deletion(-)
+
+Madelyn Olson in commit 2435341d7:
+ Added a refcount on timer events to prevent deletion of recursive timer calls
+ 2 files changed, 12 insertions(+)
+
+antirez in commit 80c906bd3:
+ Cache master without checking of deferred close flags.
+ 3 files changed, 11 insertions(+), 8 deletions(-)
+
+antirez in commit 74249be4a:
+ Track events processed while blocked globally.
+ 5 files changed, 32 insertions(+), 17 deletions(-)
+
+antirez in commit 8bf660af9:
+ Some rework of #7234.
+ 4 files changed, 77 insertions(+), 65 deletions(-)
+
+Oran Agra in commit 9da134cd8:
+ fix redis 6.0 not freeing closed connections during loading.
+ 3 files changed, 133 insertions(+), 58 deletions(-)
+
+antirez in commit f7f219a13:
+ Regression test for #7249.
+ 1 file changed, 22 insertions(+)
+
+antirez in commit 693629585:
+ rax.c updated from upstream antirez/rax.
+ 1 file changed, 4 insertions(+), 2 deletions(-)
+
+antirez in commit e3b5648df:
+ Tracking: send eviction messages when evicting entries.
+ 2 files changed, 29 insertions(+), 12 deletions(-)
+
+Oran Agra in commit 5c41802d5:
+ fix unstable replication test
+ 1 file changed, 2 insertions(+), 2 deletions(-)
+
+ShooterIT in commit a23cdbb94:
+ Redis Benchmark: Fix coredump because of double free
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+antirez in commit 1276058ea:
+ Cluster: clarify we always resolve the sender.
+ 1 file changed, 3 insertions(+), 1 deletion(-)
+
+antirez in commit 002fcde3d:
+ Cluster: refactor ping/data delay handling.
+ 1 file changed, 13 insertions(+), 11 deletions(-)
+
+antirez in commit 960186a71:
+ Cluster: introduce data_received field.
+ 2 files changed, 27 insertions(+), 10 deletions(-)
+
+antirez in commit 3672875b4:
+ stringmatchlen() should not expect null terminated strings.
+ 1 file changed, 2 insertions(+), 2 deletions(-)
+
+Brad Dunbar in commit 24e12641d:
+ Remove unreachable branch.
+ 1 file changed, 2 deletions(-)
+
+hwware in commit c7edffbd5:
+ add jemalloc-bg-thread config in redis conf
+ 1 file changed, 3 insertions(+)
+
+hwware in commit 8a9c84f4a:
+ add include guard for lolwut.h
+ 1 file changed, 6 insertions(+)
+
+antirez in commit cb683a84f:
+ Don't propagate spurious MULTI on DEBUG LOADAOF.
+ 2 files changed, 6 insertions(+), 3 deletions(-)
+
+antirez in commit 84d9766d6:
+ Dump recent backlog on master query generating errors.
+ 1 file changed, 29 insertions(+)
+
+Titouan Christophe in commit ec1e106ec:
+ make struct user anonymous (only typedefed)
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+antirez in commit e48c37316:
+ Test: --dont-clean should do first cleanup.
+ 1 file changed, 2 insertions(+), 5 deletions(-)
+
+Benjamin Sergeant in commit 1e561cfaa:
+ Add --user argument to redis-benchmark.c (ACL)
+ 1 file changed, 15 insertions(+), 2 deletions(-)
+
+antirez in commit d1af82a88:
+ Drop not needed part from #7194.
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+Muhammad Zahalqa in commit 897a360d0:
+ Fix compiler warnings on function rev(unsigned long)
+ 1 file changed, 3 insertions(+), 3 deletions(-)
+
+antirez in commit ac316d8cc:
+ Move CRC64 initialization in main().
+ 2 files changed, 1 insertion(+), 4 deletions(-)
+
+antirez in commit fc7bc3204:
+ Fix CRC64 initialization outside the Redis server itself.
+ 1 file changed, 3 insertions(+)
+
+hwware in commit a6e55c096:
+ Client Side Caching: Add Tracking Prefix Number Stats in Server Info
+ 3 files changed, 8 insertions(+)
+
+antirez in commit b062fd523:
+ Fix NetBSD build by fixing redis_set_thread_title() support.
+ 1 file changed, 4 insertions(+), 1 deletion(-)
+
+antirez in commit 4efb25d9c:
+ Rework a bit the documentation for CPU pinning.
+ 2 files changed, 18 insertions(+), 8 deletions(-)
+
+zhenwei pi in commit d6436eb7c:
+ Support setcpuaffinity on linux/bsd
+ 12 files changed, 180 insertions(+), 1 deletion(-)
+
+Guy Benoish in commit 3a441c7d9:
+ XPENDING should not update consumer's seen-time
+ 4 files changed, 33 insertions(+), 20 deletions(-)
+
+Oran Agra in commit 75addb4fe:
+ optimize memory usage of deferred replies - fixed
+ 1 file changed, 29 insertions(+)
+
+Deliang Yang in commit c57d9146f:
+ reformat code
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+Oran Agra in commit 3d3861dd8:
+ add daily github actions with libc malloc and valgrind
+ 5 files changed, 106 insertions(+), 18 deletions(-)
+
+
 ================================================================================
 Redis 6.0.1     Released Sat May 02 00:06:07 CEST 2020
 ================================================================================
diff --git a/src/version.h b/src/version.h
index 91d3566d3..ed67debe1 100644
--- a/src/version.h
+++ b/src/version.h
@@ -1 +1 @@
-#define REDIS_VERSION "6.0.1"
+#define REDIS_VERSION "6.0.2"

From 1eab62f7e59dc820356d68dfb2cf842aab297452 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Sat, 16 May 2020 17:15:35 +0200
Subject: [PATCH 166/186] Remove the client from CLOSE_ASAP list before caching
 the master.

This was broken in 1a7cd2c: we identified a crash in the CI, what
was happening before the fix should be like that:

1. The client gets in the async free list.
2. However freeClient() gets called again against the same client
   which is a master.
3. The client arrived in freeClient() with the CLOSE_ASAP flag set.
4. The master gets cached, but NOT removed from the CLOSE_ASAP linked
   list.
5. The master client that was cached was immediately removed since it
   was still in the list.
6. Redis accessed a freed cached master.

This is how the crash looked like:

=== REDIS BUG REPORT START: Cut & paste starting from here ===
1092:S 16 May 2020 11:44:09.731 # Redis 999.999.999 crashed by signal: 11
1092:S 16 May 2020 11:44:09.731 # Crashed running the instruction at: 0x447e18
1092:S 16 May 2020 11:44:09.731 # Accessing address: 0xffffffffffffffff
1092:S 16 May 2020 11:44:09.731 # Failed assertion:  (:0)

------ STACK TRACE ------
EIP:
src/redis-server 127.0.0.1:21300(readQueryFromClient+0x48)[0x447e18]

And the 0xffff address access likely comes from accessing an SDS that is
set to NULL (we go -1 offset to read the header).
---
 src/networking.c  | 21 ++++++++++++---------
 src/replication.c |  3 ---
 2 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/src/networking.c b/src/networking.c
index 6de00be09..0a04c0489 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -1134,6 +1134,16 @@ void freeClient(client *c) {
     /* Notify module system that this client auth status changed. */
     moduleNotifyUserChanged(c);
 
+    /* If this client was scheduled for async freeing we need to remove it
+     * from the queue. Note that we need to do this here, because later
+     * we may call replicationCacheMaster() and the client should already
+     * be removed from the list of clients to free. */
+    if (c->flags & CLIENT_CLOSE_ASAP) {
+        ln = listSearchKey(server.clients_to_close,c);
+        serverAssert(ln != NULL);
+        listDelNode(server.clients_to_close,ln);
+    }
+
     /* If it is our master that's beging disconnected we should make sure
      * to cache the state to try a partial resynchronization later.
      *
@@ -1142,6 +1152,7 @@ void freeClient(client *c) {
     if (server.master && c->flags & CLIENT_MASTER) {
         serverLog(LL_WARNING,"Connection with master lost.");
         if (!(c->flags & (CLIENT_PROTOCOL_ERROR|CLIENT_BLOCKED))) {
+            c->flags &= ~(CLIENT_CLOSE_ASAP|CLIENT_CLOSE_AFTER_REPLY);
             replicationCacheMaster(c);
             return;
         }
@@ -1209,15 +1220,7 @@ void freeClient(client *c) {
      * we lost the connection with the master. */
     if (c->flags & CLIENT_MASTER) replicationHandleMasterDisconnection();
 
-    /* If this client was scheduled for async freeing we need to remove it
-     * from the queue. */
-    if (c->flags & CLIENT_CLOSE_ASAP) {
-        ln = listSearchKey(server.clients_to_close,c);
-        serverAssert(ln != NULL);
-        listDelNode(server.clients_to_close,ln);
-    }
-
-    /* Remove the contribution that this client gave to our
+   /* Remove the contribution that this client gave to our
      * incrementally computed memory usage. */
     server.stat_clients_type_memory[c->client_cron_last_memory_type] -=
         c->client_cron_last_memory_usage;
diff --git a/src/replication.c b/src/replication.c
index f433d6413..b10635f25 100644
--- a/src/replication.c
+++ b/src/replication.c
@@ -2688,9 +2688,6 @@ void replicationCacheMaster(client *c) {
     /* Unlink the client from the server structures. */
     unlinkClient(c);
 
-    /* Clear flags that can create issues once we reconnect the client. */
-    c->flags &= ~(CLIENT_CLOSE_ASAP|CLIENT_CLOSE_AFTER_REPLY);
-
     /* Reset the master client so that's ready to accept new commands:
      * we want to discard te non processed query buffers and non processed
      * offsets, including pending transactions, already populated arguments,

From 7803b1148dafe65e2c7927eac7ce9e9ef729efc1 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Sat, 16 May 2020 18:11:58 +0200
Subject: [PATCH 167/186] Redis 6.0.3.

---
 00-RELEASENOTES | 8 ++++++++
 src/version.h   | 2 +-
 2 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/00-RELEASENOTES b/00-RELEASENOTES
index 1fa620bd8..7c5a75412 100644
--- a/00-RELEASENOTES
+++ b/00-RELEASENOTES
@@ -11,6 +11,14 @@ CRITICAL: There is a critical bug affecting MOST USERS. Upgrade ASAP.
 SECURITY: There are security fixes in the release.
 --------------------------------------------------------------------------------
 
+================================================================================
+Redis 6.0.3     Released Sat May 16 18:10:21 CEST 2020
+================================================================================
+
+Upgrade urgency CRITICAL: a crash introduced in 6.0.2 is now fixed.
+
+1eab62f7e Remove the client from CLOSE_ASAP list before caching the master.
+
 ================================================================================
 Redis 6.0.2     Released Fri May 15 22:24:36 CEST 2020
 ================================================================================
diff --git a/src/version.h b/src/version.h
index ed67debe1..8ae7b0cb3 100644
--- a/src/version.h
+++ b/src/version.h
@@ -1 +1 @@
-#define REDIS_VERSION "6.0.2"
+#define REDIS_VERSION "6.0.3"

From 50fb85a4893fcfea4c49b4979fb03875ffaa4136 Mon Sep 17 00:00:00 2001
From: John Sully <john@csquare.ca>
Date: Wed, 20 May 2020 15:57:08 -0400
Subject: [PATCH 168/186] Override missing new, fix new/delete mismatch

Former-commit-id: 712d825c233c3ebbc49b22b8c1865916ca9dc3bc
---
 src/new.cpp | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/src/new.cpp b/src/new.cpp
index 33693da7a..586c4fefd 100644
--- a/src/new.cpp
+++ b/src/new.cpp
@@ -1,6 +1,7 @@
 #include <cstddef>  // std::size_t
 #include "server.h"
 #include "new.h"
+#include <new>
 
 #ifdef SANITIZE
 void *operator new(size_t size, enum MALLOC_CLASS mclass)
@@ -21,6 +22,11 @@ void *operator new(size_t size, enum MALLOC_CLASS mclass)
     return zmalloc(size, mclass);
 }
 
+void *operator new(std::size_t size, const std::nothrow_t &)
+{
+    return zmalloc(size, MALLOC_LOCAL);
+}
+
 void operator delete(void * p) noexcept
 {
     zfree(p);
@@ -31,4 +37,4 @@ void operator delete(void *p, std::size_t) noexcept
     zfree(p);
 }
 
-#endif
\ No newline at end of file
+#endif

From 8a14bb4b104738170e2dbfe81c7209db874aa98c Mon Sep 17 00:00:00 2001
From: John Sully <john@csquare.ca>
Date: Thu, 21 May 2020 23:24:34 -0400
Subject: [PATCH 169/186] build failure on macos

Former-commit-id: c10aa3b8b0811da9931745695c5de06fdfa99fe3
---
 src/config.h | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/config.h b/src/config.h
index 659b0aff3..431491953 100644
--- a/src/config.h
+++ b/src/config.h
@@ -241,6 +241,9 @@ void setproctitle(const char *fmt, ...);
 #define redis_set_thread_title(name) pthread_set_name_np(pthread_self(), name)
 #else
 #if (defined __APPLE__ && defined(MAC_OS_X_VERSION_10_7))
+#ifdef __cplusplus
+extern "C" 
+#endif
 int pthread_setname_np(const char *name);
 #include <pthread.h>
 #define redis_set_thread_title(name) pthread_setname_np(name)

From 08a2facf13bce9c9d893b2c75edfd64d57cedb3b Mon Sep 17 00:00:00 2001
From: John Sully <john@csquare.ca>
Date: Thu, 21 May 2020 23:25:50 -0400
Subject: [PATCH 170/186] Missing noexcept

Former-commit-id: f214ada545f810f8a9095753984b75c438cd6464
---
 src/new.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/new.cpp b/src/new.cpp
index 586c4fefd..3b3a0755b 100644
--- a/src/new.cpp
+++ b/src/new.cpp
@@ -22,7 +22,7 @@ void *operator new(size_t size, enum MALLOC_CLASS mclass)
     return zmalloc(size, mclass);
 }
 
-void *operator new(std::size_t size, const std::nothrow_t &)
+void *operator new(std::size_t size, const std::nothrow_t &) noexcept
 {
     return zmalloc(size, MALLOC_LOCAL);
 }

From bc1f7d6d454811d7008f5c4303339f169abcb2e6 Mon Sep 17 00:00:00 2001
From: John Sully <john@csquare.ca>
Date: Fri, 22 May 2020 14:03:29 -0400
Subject: [PATCH 171/186] Update README.md

Former-commit-id: a25cf15ecd6e7a013f4b653b3b103c9e9227af3a
---
 README.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/README.md b/README.md
index 06d85c86e..88f5f0f7f 100644
--- a/README.md
+++ b/README.md
@@ -3,6 +3,8 @@
 [![Join the chat at https://gitter.im/KeyDB/community](https://badges.gitter.im/KeyDB/community.svg)](https://gitter.im/KeyDB/community?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
 [![StackShare](http://img.shields.io/badge/tech-stack-0690fa.svg?style=flat)](https://stackshare.io/eq-alpha-technology-inc/eq-alpha-technology-inc)
 
+##### New! Want to extend KeyDB with Javascript?  Try [ModJS](https://github.com/JohnSully/ModJS)
+
 ##### Need Help? Check out our extensive [documentation](https://docs.keydb.dev).
 
 ##### Have feedback?  Take our quick survey: https://www.surveymonkey.com/r/Y9XNS93

From 23b4dc503aeb38f1bb3e21bb845cae5cac63a8d3 Mon Sep 17 00:00:00 2001
From: ShooterIT <wangyuancode@163.com>
Date: Thu, 21 May 2020 21:00:21 +0800
Subject: [PATCH 172/186] Fix reply bytes calculation error

Fix #7275.


Former-commit-id: a12210c87561fed5522c9a6e8db6d8606cabe3bf
---
 src/networking.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/networking.cpp b/src/networking.cpp
index 13e7f4da3..579e81561 100644
--- a/src/networking.cpp
+++ b/src/networking.cpp
@@ -612,7 +612,7 @@ void trimReplyUnusedTailSpace(client *c) {
         /* take over the allocation's internal fragmentation (at least for
          * memory usage tracking) */
         tail->size = zmalloc_usable(tail) - sizeof(clientReplyBlock);
-        c->reply_bytes += tail->size - old_size;
+        c->reply_bytes = c->reply_bytes + tail->size - old_size;
         listNodeValue(ln) = tail;
     }
 }

From 5d9afccfdd39bc40aa946fd3f2f5f70bd9c0625a Mon Sep 17 00:00:00 2001
From: John Sully <john@csquare.ca>
Date: Sat, 23 May 2020 15:15:42 -0400
Subject: [PATCH 173/186] Diskless replication handler needs to run on the main
 thread

Former-commit-id: 091af6400348b7716f7fe0bc845bf7f16d922815
---
 src/rdb.cpp         | 10 ++++++----
 src/replication.cpp | 10 ++++++----
 2 files changed, 12 insertions(+), 8 deletions(-)

diff --git a/src/rdb.cpp b/src/rdb.cpp
index abaa3d608..97f8aebc0 100644
--- a/src/rdb.cpp
+++ b/src/rdb.cpp
@@ -2429,7 +2429,7 @@ int rdbLoadRio(rio *rdb, int rdbflags, rdbSaveInfo *rsi) {
                 val = nullptr;
             }
         }
-        
+
         if (g_pserver->key_load_delay)
             usleep(g_pserver->key_load_delay);
 
@@ -2701,9 +2701,11 @@ int rdbSaveToSlavesSockets(rdbSaveInfo *rsi) {
             g_pserver->rdb_child_pid = childpid;
             g_pserver->rdb_child_type = RDB_CHILD_TYPE_SOCKET;
             close(g_pserver->rdb_pipe_write); /* close write in parent so that it can detect the close on the child. */
-            if (aeCreateFileEvent(serverTL->el, g_pserver->rdb_pipe_read, AE_READABLE, rdbPipeReadHandler,NULL) == AE_ERR) {
-                serverPanic("Unrecoverable error creating server.rdb_pipe_read file event.");
-            }
+            aePostFunction(g_pserver->rgthreadvar[IDX_EVENT_LOOP_MAIN].el, []{
+                if (aeCreateFileEvent(serverTL->el, g_pserver->rdb_pipe_read, AE_READABLE, rdbPipeReadHandler,NULL) == AE_ERR) {
+                    serverPanic("Unrecoverable error creating server.rdb_pipe_read file event.");
+                }
+            });
         }
         return (childpid == -1) ? C_ERR : C_OK;
     }
diff --git a/src/replication.cpp b/src/replication.cpp
index 766a9ff67..48b9f21f8 100644
--- a/src/replication.cpp
+++ b/src/replication.cpp
@@ -1345,9 +1345,11 @@ void rdbPipeWriteHandlerConnRemoved(struct connection *conn) {
     g_pserver->rdb_pipe_numconns_writing--;
     /* if there are no more writes for now for this conn, or write error: */
     if (g_pserver->rdb_pipe_numconns_writing == 0) {
-        if (aeCreateFileEvent(serverTL->el, g_pserver->rdb_pipe_read, AE_READABLE, rdbPipeReadHandler,NULL) == AE_ERR) {
-            serverPanic("Unrecoverable error creating g_pserver->rdb_pipe_read file event.");
-        }
+        aePostFunction(g_pserver->rgthreadvar[IDX_EVENT_LOOP_MAIN].el, []{
+            if (aeCreateFileEvent(serverTL->el, g_pserver->rdb_pipe_read, AE_READABLE, rdbPipeReadHandler,NULL) == AE_ERR) {
+                serverPanic("Unrecoverable error creating g_pserver->rdb_pipe_read file event.");
+            }
+        });
     }
 }
 
@@ -1398,11 +1400,11 @@ void RdbPipeCleanup() {
 void rdbPipeReadHandler(struct aeEventLoop *eventLoop, int fd, void *clientData, int mask) {
     UNUSED(mask);
     UNUSED(clientData);
-    UNUSED(eventLoop);
     int i;
     if (!g_pserver->rdb_pipe_buff)
         g_pserver->rdb_pipe_buff = (char*)zmalloc(PROTO_IOBUF_LEN);
     serverAssert(g_pserver->rdb_pipe_numconns_writing==0);
+    serverAssert(eventLoop == g_pserver->rgthreadvar[IDX_EVENT_LOOP_MAIN].el);
 
     while (1) {
         g_pserver->rdb_pipe_bufflen = read(fd, g_pserver->rdb_pipe_buff, PROTO_IOBUF_LEN);

From 18b8a83ecb11f243bbc35d0a54b869bb28e778ec Mon Sep 17 00:00:00 2001
From: John Sully <john@csquare.ca>
Date: Sat, 23 May 2020 16:03:24 -0400
Subject: [PATCH 174/186] During RDB load we can only handle clients on the
 main thread as our workers are not yet created.  This change also unifies the
 accept logic across the socket types

Former-commit-id: 59cda9cfb96eca55c4b445309970bd95277ebc5c
---
 src/networking.cpp  | 123 +++++++++++++++++++-------------------------
 src/replication.cpp |   1 +
 src/server.cpp      |  13 +++--
 3 files changed, 64 insertions(+), 73 deletions(-)

diff --git a/src/networking.cpp b/src/networking.cpp
index 579e81561..af97f229b 100644
--- a/src/networking.cpp
+++ b/src/networking.cpp
@@ -1283,11 +1283,60 @@ static void acceptCommonHandler(connection *conn, int flags, char *ip, int iel)
     }
 }
 
+void acceptOnThread(connection *conn, int flags, char *cip)
+{
+    int ielCur = ielFromEventLoop(serverTL->el);
+
+    int ielTarget = 0;
+    if (g_pserver->loading)
+    {
+        ielTarget = IDX_EVENT_LOOP_MAIN;    // During load only the main thread is active
+    }
+    else if (g_fTestMode)
+    {
+        // On test mode we don't want any bunching of clients
+        while (cserver.cthreads > 1 && ielTarget == IDX_EVENT_LOOP_MAIN)
+            ielTarget = rand() % cserver.cthreads;
+    }
+    else
+    {
+        ielTarget = chooseBestThreadForAccept();
+    }
+
+    rgacceptsInFlight[ielTarget].fetch_add(1, std::memory_order_relaxed);
+    if (ielTarget != ielCur)
+    {
+        char *szT = nullptr;
+        if (cip != nullptr)
+        {
+            szT = (char*)zmalloc(NET_IP_STR_LEN, MALLOC_LOCAL);
+            memcpy(szT, cip, NET_IP_STR_LEN);
+        }
+        int res = aePostFunction(g_pserver->rgthreadvar[ielTarget].el, [conn, flags, ielTarget, szT]{
+            acceptCommonHandler(conn,flags,szT,ielTarget);
+            if (!g_fTestMode && !g_pserver->loading)
+                rgacceptsInFlight[ielTarget].fetch_sub(1, std::memory_order_relaxed);
+            zfree(szT);
+        });
+
+        if (res == AE_OK)
+            return;
+        // If res != AE_OK we can still try to accept on the local thread
+    }
+    if (!g_fTestMode && !g_pserver->loading)
+        rgacceptsInFlight[ielTarget].fetch_sub(1, std::memory_order_relaxed);
+
+    aeAcquireLock();
+    acceptCommonHandler(conn,flags,cip,ielCur);
+    aeReleaseLock();
+}
+
 void acceptTcpHandler(aeEventLoop *el, int fd, void *privdata, int mask) {
     int cport, cfd, max = MAX_ACCEPTS_PER_CALL;
     char cip[NET_IP_STR_LEN];
     UNUSED(mask);
     UNUSED(privdata);
+    UNUSED(el);
 
     while(max--) {
         cfd = anetTcpAccept(serverTL->neterr, fd, cip, sizeof(cip), &cport);
@@ -1298,55 +1347,8 @@ void acceptTcpHandler(aeEventLoop *el, int fd, void *privdata, int mask) {
             return;
         }
         serverLog(LL_VERBOSE,"Accepted %s:%d", cip, cport);
-        int ielCur = ielFromEventLoop(el);
 
-        if (!g_fTestMode)
-        {
-            {
-            int ielTarget = chooseBestThreadForAccept();
-            rgacceptsInFlight[ielTarget].fetch_add(1, std::memory_order_relaxed);
-            if (ielTarget != ielCur)
-            {
-                char *szT = (char*)zmalloc(NET_IP_STR_LEN, MALLOC_LOCAL);
-                memcpy(szT, cip, NET_IP_STR_LEN);
-                int res = aePostFunction(g_pserver->rgthreadvar[ielTarget].el, [cfd, ielTarget, szT]{
-                    acceptCommonHandler(connCreateAcceptedSocket(cfd),0,szT,ielTarget);
-                    rgacceptsInFlight[ielTarget].fetch_sub(1, std::memory_order_relaxed);
-                    zfree(szT);
-                });
-
-                if (res == AE_OK)
-                    continue;
-            }
-            rgacceptsInFlight[ielTarget].fetch_sub(1, std::memory_order_relaxed);
-            }
-
-        LLocalThread:
-            aeAcquireLock();
-            acceptCommonHandler(connCreateAcceptedSocket(cfd),0,cip,ielCur);
-            aeReleaseLock();
-        }
-        else
-        {
-            // In test mode we want a good distribution among threads and avoid the main thread
-            //  since the main thread is most likely to work
-            int iel = IDX_EVENT_LOOP_MAIN;
-            while (cserver.cthreads > 1 && iel == IDX_EVENT_LOOP_MAIN)
-                iel = rand() % cserver.cthreads;
-            if (iel == ielFromEventLoop(el))
-                goto LLocalThread;
-            char *szT = (char*)zmalloc(NET_IP_STR_LEN, MALLOC_LOCAL);
-            memcpy(szT, cip, NET_IP_STR_LEN);
-            int res = aePostFunction(g_pserver->rgthreadvar[iel].el, [cfd, iel, szT]{
-                acceptCommonHandler(connCreateAcceptedSocket(cfd),0,szT,iel);
-                zfree(szT);
-            });
-            if (res != AE_OK)
-            {
-                zfree(szT);
-                goto LLocalThread;
-            }
-        }
+        acceptOnThread(connCreateAcceptedSocket(cfd), 0, cip);
     }
 }
 
@@ -1357,7 +1359,6 @@ void acceptTLSHandler(aeEventLoop *el, int fd, void *privdata, int mask) {
     UNUSED(mask);
     UNUSED(privdata);
 
-    int ielCur = ielFromEventLoop(el);
     while(max--) {
         cfd = anetTcpAccept(serverTL->neterr, fd, cip, sizeof(cip), &cport);
         if (cfd == ANET_ERR) {
@@ -1367,9 +1368,8 @@ void acceptTLSHandler(aeEventLoop *el, int fd, void *privdata, int mask) {
             return;
         }
         serverLog(LL_VERBOSE,"Accepted %s:%d", cip, cport);
-        aeAcquireLock();
-        acceptCommonHandler(connCreateAcceptedTLS(cfd, g_pserver->tls_auth_clients),0,cip,ielCur);
-        aeReleaseLock();
+
+        acceptOnThread(connCreateAcceptedTLS(cfd, g_pserver->tls_auth_clients), 0, cip);
     }
 }
 
@@ -1379,7 +1379,6 @@ void acceptUnixHandler(aeEventLoop *el, int fd, void *privdata, int mask) {
     UNUSED(mask);
     UNUSED(privdata);
 
-    int iel = ielFromEventLoop(el);
     while(max--) {
         cfd = anetUnixAccept(serverTL->neterr, fd);
         if (cfd == ANET_ERR) {
@@ -1390,23 +1389,7 @@ void acceptUnixHandler(aeEventLoop *el, int fd, void *privdata, int mask) {
         }
         serverLog(LL_VERBOSE,"Accepted connection to %s", g_pserver->unixsocket);
 
-        aeAcquireLock();
-        int ielTarget = rand() % cserver.cthreads;
-        if (ielTarget == iel)
-        {
-        LLocalThread:
-            acceptCommonHandler(connCreateAcceptedSocket(cfd),CLIENT_UNIX_SOCKET,NULL,iel);
-        }
-        else
-        {
-            int res = aePostFunction(g_pserver->rgthreadvar[ielTarget].el, [cfd, ielTarget]{
-                acceptCommonHandler(connCreateAcceptedSocket(cfd),CLIENT_UNIX_SOCKET,NULL,ielTarget);
-            });
-            if (res != AE_OK)
-                goto LLocalThread;
-        }
-        aeReleaseLock();
-        
+        acceptOnThread(connCreateAcceptedSocket(cfd),CLIENT_UNIX_SOCKET,NULL);
     }
 }
 
diff --git a/src/replication.cpp b/src/replication.cpp
index 48b9f21f8..a0ca766fc 100644
--- a/src/replication.cpp
+++ b/src/replication.cpp
@@ -1449,6 +1449,7 @@ void rdbPipeReadHandler(struct aeEventLoop *eventLoop, int fd, void *clientData,
                 continue;
 
             client *slave = (client*)connGetPrivateData(conn);
+            std::unique_lock<fastlock> ul(slave->lock);
             // Normally it would be bug to talk a client conn from a different thread, but here we know nobody else will
             //  be sending anything while in this replication state so it is OK
             if ((nwritten = connWrite(conn, g_pserver->rdb_pipe_buff, g_pserver->rdb_pipe_bufflen)) == -1) {
diff --git a/src/server.cpp b/src/server.cpp
index d7dcdaebf..8d2a10129 100644
--- a/src/server.cpp
+++ b/src/server.cpp
@@ -2923,9 +2923,9 @@ static void initNetworkingThread(int iel, int fReusePort)
 
 static void initNetworking(int fReusePort)
 {
-    int celListen = (fReusePort) ? cserver.cthreads : 1;
-    for (int iel = 0; iel < celListen; ++iel)
-        initNetworkingThread(iel, fReusePort);
+    // We only initialize the main thread here, since RDB load is a special case that processes
+    //  clients before our server threads are launched.
+    initNetworkingThread(IDX_EVENT_LOOP_MAIN, fReusePort);
 
     /* Open the listening Unix domain socket. */
     if (g_pserver->unixsocket != NULL) {
@@ -5298,6 +5298,13 @@ void *workerThreadMain(void *parg)
     serverLog(LOG_INFO, "Thread %d alive.", iel);
     serverTL = g_pserver->rgthreadvar+iel;  // set the TLS threadsafe global
 
+    if (iel != IDX_EVENT_LOOP_MAIN)
+    {
+        aeAcquireLock();
+        initNetworkingThread(iel, cserver.cthreads > 1);
+        aeReleaseLock();
+    }
+
     moduleAcquireGIL(true); // Normally afterSleep acquires this, but that won't be called on the first run
     aeEventLoop *el = g_pserver->rgthreadvar[iel].el;
     try

From de376acb15281519cb5dfda93812ce317d884a56 Mon Sep 17 00:00:00 2001
From: John Sully <john@csquare.ca>
Date: Sun, 24 May 2020 01:53:43 -0400
Subject: [PATCH 175/186] race calculating client overhead

Former-commit-id: 2d27c8ecdb5242284301f135b1d542f9ead5b8f2
---
 src/evict.cpp | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/evict.cpp b/src/evict.cpp
index ffd241446..23d873975 100644
--- a/src/evict.cpp
+++ b/src/evict.cpp
@@ -33,6 +33,7 @@
 #include "server.h"
 #include "bio.h"
 #include "atomicvar.h"
+#include <mutex>
 
 /* ----------------------------------------------------------------------------
  * Data structures
@@ -375,6 +376,7 @@ size_t freeMemoryGetNotCountedMemory(void) {
         listRewind(g_pserver->slaves,&li);
         while((ln = listNext(&li))) {
             client *replica = (client*)listNodeValue(ln);
+            std::unique_lock<fastlock>(replica->lock);
             overhead += getClientOutputBufferMemoryUsage(replica);
         }
     }

From 926da1f89988708d24d70954995bcee8f62d70f2 Mon Sep 17 00:00:00 2001
From: John Sully <john@csquare.ca>
Date: Sun, 24 May 2020 01:56:23 -0400
Subject: [PATCH 176/186] If we request shutdown from a client on a different
 thread than a blocking lua script we fail to actually shutdown until the
 script completes

Former-commit-id: 1ca6c1dd56131bc28fe9f831e2b9cab7adb846a3
---
 src/networking.cpp | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/networking.cpp b/src/networking.cpp
index af97f229b..e3e0b0e00 100644
--- a/src/networking.cpp
+++ b/src/networking.cpp
@@ -3473,7 +3473,12 @@ void processEventsWhileBlocked(int iel) {
     AeLocker locker;
     locker.arm(nullptr);
     locker.release();
+
     for (client *c : vecclients)
         c->lock.lock();
+
+    // If a different thread processed the shutdown we need to abort the lua command or we will hang
+    if (serverTL->el->stop)
+        throw ShutdownException();
 }
 

From 51976e2be885923b1f8ab5f825d0095788bdcbca Mon Sep 17 00:00:00 2001
From: John Sully <john@csquare.ca>
Date: Sun, 24 May 2020 01:57:18 -0400
Subject: [PATCH 177/186] Incorrect memory order on weak ordered architectures

Former-commit-id: 90162da1a6c025d148a0d1761ae4d5fbc9bfc0ef
---
 src/server.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/server.h b/src/server.h
index 753c641c0..9274538c5 100644
--- a/src/server.h
+++ b/src/server.h
@@ -801,7 +801,7 @@ public:
     void setrefcount(unsigned ref);
     unsigned getrefcount(std::memory_order order = std::memory_order_relaxed) const { return (refcount.load(order) & ~(1U << 31)); }
     void addref() const { refcount.fetch_add(1, std::memory_order_relaxed); }
-    unsigned release() const { return refcount.fetch_sub(1, std::memory_order_relaxed) & ~(1U << 31); }
+    unsigned release() const { return refcount.fetch_sub(1, std::memory_order_seq_cst) & ~(1U << 31); }
 } robj;
 static_assert(sizeof(redisObject) == 24, "object size is critical, don't increase");
 
@@ -1682,7 +1682,7 @@ struct redisServer {
     std::atomic<uint64_t> next_client_id; /* Next client unique ID. Incremental. */
     int protected_mode;         /* Don't accept external connections. */
     long long events_processed_while_blocked; /* processEventsWhileBlocked() */
-    
+
     /* RDB / AOF loading information */
     std::atomic<int> loading;                /* We are loading data from disk if true */
     off_t loading_total_bytes;

From cc080de22181035f2b752b32fc1f1b445944ec3b Mon Sep 17 00:00:00 2001
From: John Sully <john@csquare.ca>
Date: Sun, 24 May 2020 01:57:49 -0400
Subject: [PATCH 178/186] Incorrect locking in replication error paths

Former-commit-id: 1c6b57314747787958ad215826296627f9050f59
---
 src/replication.cpp | 22 +++++++++++++++++++++-
 1 file changed, 21 insertions(+), 1 deletion(-)

diff --git a/src/replication.cpp b/src/replication.cpp
index a0ca766fc..36a6cb109 100644
--- a/src/replication.cpp
+++ b/src/replication.cpp
@@ -33,6 +33,7 @@
 #include "server.h"
 #include "cluster.h"
 #include "bio.h"
+#include "aelocker.h"
 
 #include <sys/time.h>
 #include <unistd.h>
@@ -1284,6 +1285,7 @@ void sendBulkToSlave(connection *conn) {
     serverAssert(FCorrectThread(replica));
     char buf[PROTO_IOBUF_LEN];
     ssize_t nwritten, buflen;
+    AeLocker aeLock;
     std::unique_lock<fastlock> ul(replica->lock);
 
     /* Before sending the RDB file, we send the preamble as configured by the
@@ -1295,6 +1297,8 @@ void sendBulkToSlave(connection *conn) {
             serverLog(LL_VERBOSE,
                 "Write error sending RDB preamble to replica: %s",
                 connGetLastError(conn));
+            ul.unlock();
+            aeLock.arm(nullptr);
             freeClient(replica);
             return;
         }
@@ -1315,6 +1319,8 @@ void sendBulkToSlave(connection *conn) {
     if (buflen <= 0) {
         serverLog(LL_WARNING,"Read error sending DB to replica: %s",
             (buflen == 0) ? "premature EOF" : strerror(errno));
+        ul.unlock();
+        aeLock.arm(nullptr);
         freeClient(replica);
         return;
     }
@@ -1322,6 +1328,8 @@ void sendBulkToSlave(connection *conn) {
         if (connGetState(conn) != CONN_STATE_CONNECTED) {
             serverLog(LL_WARNING,"Write error sending DB to replica: %s",
                 connGetLastError(conn));
+            ul.unlock();
+            aeLock.arm(nullptr);
             freeClient(replica);
         }
         return;
@@ -1516,6 +1524,8 @@ void updateSlavesWaitingBgsave(int bgsaveerr, int type)
     while((ln = listNext(&li))) {
         client *replica = (client*)ln->value;
 
+        std::unique_lock<fastlock> ul(replica->lock);
+
         if (replica->replstate == SLAVE_STATE_WAIT_BGSAVE_START) {
             startbgsave = 1;
             mincapa = (mincapa == -1) ? replica->slave_capa :
@@ -1562,6 +1572,7 @@ void updateSlavesWaitingBgsave(int bgsaveerr, int type)
                 replica->repl_ack_time = g_pserver->unixtime; /* Timeout otherwise. */
             } else {
                 if (bgsaveerr != C_OK) {
+                    ul.unlock();
                     if (FCorrectThread(replica))
                         freeClient(replica);
                     else
@@ -1571,6 +1582,7 @@ void updateSlavesWaitingBgsave(int bgsaveerr, int type)
                 }
                 if ((replica->repldbfd = open(g_pserver->rdb_filename,O_RDONLY)) == -1 ||
                     redis_fstat(replica->repldbfd,&buf) == -1) {
+                    ul.unlock();
                     if (FCorrectThread(replica))
                         freeClient(replica);
                     else
@@ -1588,6 +1600,7 @@ void updateSlavesWaitingBgsave(int bgsaveerr, int type)
                 {
                     connSetWriteHandler(replica->conn,NULL);
                     if (connSetWriteHandler(replica->conn,sendBulkToSlave) == C_ERR) {
+                        ul.unlock();
                         freeClient(replica);
                         continue;
                     }
@@ -3808,6 +3821,7 @@ void replicationCron(void) {
         listRewind(g_pserver->slaves,&li);
         while((ln = listNext(&li))) {
             client *replica = (client*)ln->value;
+            std::unique_lock<fastlock> ul(replica->lock);
 
             if (replica->replstate != SLAVE_STATE_ONLINE) continue;
             if (replica->flags & CLIENT_PRE_PSYNC) continue;
@@ -3816,9 +3830,15 @@ void replicationCron(void) {
                 serverLog(LL_WARNING, "Disconnecting timedout replica: %s",
                     replicationGetSlaveName(replica));
                 if (FCorrectThread(replica))
-                    freeClient(replica);
+                {
+                    ul.release();
+                    if (!freeClient(replica))
+                        replica->lock.unlock(); // we didn't free so we have undo the lock we just released
+                }
                 else
+                {
                     freeClientAsync(replica);
+                }
             }
         }
     }

From dc27b95a4703bedd9434e008e50957a043069f71 Mon Sep 17 00:00:00 2001
From: John Sully <john@csquare.ca>
Date: Sun, 24 May 2020 01:58:33 -0400
Subject: [PATCH 179/186] Fix ASAN races

Former-commit-id: 8665f250314c4e3de32f3b31a2c0158908ef982b
---
 src/networking.cpp | 14 ++++++--------
 1 file changed, 6 insertions(+), 8 deletions(-)

diff --git a/src/networking.cpp b/src/networking.cpp
index e3e0b0e00..dbef33655 100644
--- a/src/networking.cpp
+++ b/src/networking.cpp
@@ -80,6 +80,7 @@ int listMatchObjects(void *a, void *b) {
 /* This function links the client to the global linked list of clients.
  * unlinkClient() does the opposite, among other things. */
 void linkClient(client *c) {
+    serverAssert(GlobalLocksAcquired());
     listAddNodeTail(g_pserver->clients,c);
     /* Note that we remember the linked list node where the client is stored,
      * this way removing the client in unlinkClient() will not require
@@ -1957,12 +1958,9 @@ int handleClientsWithPendingWrites(int iel, int aof_state) {
         }
     }
 
-    if (listLength(serverTL->clients_pending_asyncwrite))
-    {
-        AeLocker locker;
-        locker.arm(nullptr);
-        ProcessPendingAsyncWrites();
-    }
+    AeLocker locker;
+    locker.arm(nullptr);
+    ProcessPendingAsyncWrites();
 
     return processed;
 }
@@ -2298,13 +2296,13 @@ void commandProcessed(client *c) {
      * part of the replication stream, will be propagated to the
      * sub-replicas and to the replication backlog. */
     if (c->flags & CLIENT_MASTER) {
+        AeLocker ae;
+            ae.arm(c);
         long long applied = c->reploff - prev_offset;
         long long prev_master_repl_meaningful_offset = g_pserver->master_repl_meaningful_offset;
         if (applied) {
             if (!g_pserver->fActiveReplica)
             {
-                AeLocker ae;
-                ae.arm(c);
                 replicationFeedSlavesFromMasterStream(g_pserver->slaves,
                     c->pending_querybuf, applied);
             }

From 78b647bd6f436df94570e59de9493ce19ddf6277 Mon Sep 17 00:00:00 2001
From: John Sully <john@csquare.ca>
Date: Sun, 24 May 2020 02:45:11 -0400
Subject: [PATCH 180/186] race in connection state when setting a handler on a
 different thread

Former-commit-id: 42c7677690c434168135992429f7521239475ef8
---
 src/connection.h | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/connection.h b/src/connection.h
index 10f9e322b..f5624b596 100644
--- a/src/connection.h
+++ b/src/connection.h
@@ -31,6 +31,8 @@
 #ifndef __REDIS_CONNECTION_H
 #define __REDIS_CONNECTION_H
 
+#include <atomic>
+
 #define CONN_INFO_LEN   32
 
 struct aeEventLoop;
@@ -70,7 +72,7 @@ typedef struct ConnectionType {
 
 struct connection {
     ConnectionType *type;
-    ConnectionState state;
+    std::atomic<ConnectionState> state;
     short int flags;
     short int refs;
     int last_errno;

From f517eaf182ad44f4451142e98b604fe25555de96 Mon Sep 17 00:00:00 2001
From: John Sully <john@csquare.ca>
Date: Sun, 24 May 2020 02:45:55 -0400
Subject: [PATCH 181/186] remove unnecessary cplusplus guard

Former-commit-id: 3b8463f9d20162369cffd9ed347865c5c8049f40
---
 src/rdb-s3.cpp | 2 --
 1 file changed, 2 deletions(-)

diff --git a/src/rdb-s3.cpp b/src/rdb-s3.cpp
index 0171271b4..98c29a6d1 100644
--- a/src/rdb-s3.cpp
+++ b/src/rdb-s3.cpp
@@ -1,6 +1,4 @@
-extern "C" {
 #include "rio.h"
-}
 #include "server.h"
 #include <unistd.h>
 #include <sys/wait.h>

From b892d3defff00a121435a9a4e7f1b6d9f10a859c Mon Sep 17 00:00:00 2001
From: John Sully <john@csquare.ca>
Date: Sun, 24 May 2020 02:46:39 -0400
Subject: [PATCH 182/186] Add standardized way to post client dependent
 functions

Former-commit-id: fb50163b47cf660911dc7f67809c15ba1394613a
---
 src/replication.cpp | 18 ++----------------
 src/server.cpp      |  9 +++++++++
 src/server.h        |  3 +++
 3 files changed, 14 insertions(+), 16 deletions(-)

diff --git a/src/replication.cpp b/src/replication.cpp
index 36a6cb109..6b18d27ac 100644
--- a/src/replication.cpp
+++ b/src/replication.cpp
@@ -1478,7 +1478,7 @@ void rdbPipeReadHandler(struct aeEventLoop *eventLoop, int fd, void *clientData,
              * setup write handler (and disable pipe read handler, below) */
             if (nwritten != g_pserver->rdb_pipe_bufflen) {
                 g_pserver->rdb_pipe_numconns_writing++;
-                aePostFunction(g_pserver->rgthreadvar[slave->iel].el, [conn] {
+                slave->postFunction([conn](client *) {
                     connSetWriteHandler(conn, rdbPipeWriteHandler);
                 });
             }
@@ -1607,21 +1607,7 @@ void updateSlavesWaitingBgsave(int bgsaveerr, int type)
                 }
                 else
                 {
-                    aePostFunction(g_pserver->rgthreadvar[replica->iel].el, [replica] {
-                        // Because the client could have been closed while the lambda waited to run we need to
-			            // verify the replica is still connected
-                        listIter li;
-                        listNode *ln;
-                        listRewind(g_pserver->slaves,&li);
-                        bool fFound = false;
-                        while ((ln = listNext(&li))) {
-                            if (listNodeValue(ln) == replica) {
-                                fFound = true;
-                                break;
-                            }
-                        }
-                        if (!fFound)
-                            return;
+                    replica->postFunction([](client *replica) {
                         connSetWriteHandler(replica->conn,NULL);
                         if (connSetWriteHandler(replica->conn,sendBulkToSlave) == C_ERR) {
                             freeClient(replica);
diff --git a/src/server.cpp b/src/server.cpp
index 8d2a10129..ddebe3ac8 100644
--- a/src/server.cpp
+++ b/src/server.cpp
@@ -3887,6 +3887,15 @@ int processCommand(client *c, int callFlags) {
     return C_OK;
 }
 
+bool client::postFunction(std::function<void(client *)> fn) {
+    this->casyncOpsPending++;
+    return aePostFunction(g_pserver->rgthreadvar[this->iel].el, [this, fn]{
+        std::lock_guard<decltype(this->lock)> lock(this->lock);
+        --casyncOpsPending;
+        fn(this);
+    }) == AE_OK;
+}
+
 /*================================== Shutdown =============================== */
 
 /* Close listening sockets. Also unlink the unix domain socket if
diff --git a/src/server.h b/src/server.h
index 9274538c5..e3c62796b 100644
--- a/src/server.h
+++ b/src/server.h
@@ -1335,6 +1335,9 @@ typedef struct client {
     int iel; /* the event loop index we're registered with */
     struct fastlock lock;
     int master_error;
+
+    // post a function from a non-client thread to run on its client thread
+    bool postFunction(std::function<void(client *)> fn);
 } client;
 
 struct saveparam {

From 23e160127604f4f168757f630721f332f0db3493 Mon Sep 17 00:00:00 2001
From: John Sully <john@csquare.ca>
Date: Sun, 24 May 2020 11:19:23 -0400
Subject: [PATCH 183/186] Hold the AE lock until we process the repl backlog
 after a command

Former-commit-id: 4893eedacbf5f9210d02361ed07b9beee0c9f655
---
 src/networking.cpp | 3 ++-
 src/server.cpp     | 3 +--
 src/server.h       | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/networking.cpp b/src/networking.cpp
index dbef33655..58ca4f86b 100644
--- a/src/networking.cpp
+++ b/src/networking.cpp
@@ -2326,7 +2326,8 @@ void commandProcessed(client *c) {
 int processCommandAndResetClient(client *c, int flags) {
     int deadclient = 0;
     serverTL->current_client = c;
-    if (processCommand(c, flags) == C_OK) {
+    AeLocker locker;
+    if (processCommand(c, flags, locker) == C_OK) {
         commandProcessed(c);
     }
     if (serverTL->current_client == NULL) deadclient = 1;
diff --git a/src/server.cpp b/src/server.cpp
index ddebe3ac8..8566dd7f0 100644
--- a/src/server.cpp
+++ b/src/server.cpp
@@ -3621,8 +3621,7 @@ void call(client *c, int flags) {
  * If C_OK is returned the client is still alive and valid and
  * other operations can be performed by the caller. Otherwise
  * if C_ERR is returned the client was destroyed (i.e. after QUIT). */
-int processCommand(client *c, int callFlags) {
-    AeLocker locker;
+int processCommand(client *c, int callFlags, AeLocker &locker) {
     AssertCorrectThread(c);
 
     if (moduleHasCommandFilters())
diff --git a/src/server.h b/src/server.h
index e3c62796b..628e1d3b9 100644
--- a/src/server.h
+++ b/src/server.h
@@ -2542,7 +2542,7 @@ int getMaxmemoryState(size_t *total, size_t *logical, size_t *tofree, float *lev
 size_t freeMemoryGetNotCountedMemory();
 int freeMemoryIfNeeded(void);
 int freeMemoryIfNeededAndSafe(void);
-int processCommand(client *c, int callFlags);
+int processCommand(client *c, int callFlags, class AeLocker &locker);
 void setupSignalHandlers(void);
 struct redisCommand *lookupCommand(sds name);
 struct redisCommand *lookupCommandByCString(const char *s);

From 70f3eab882a70b333c6fb7ff1ac9ccd00f5baa0e Mon Sep 17 00:00:00 2001
From: John Sully <john@csquare.ca>
Date: Sun, 24 May 2020 11:19:53 -0400
Subject: [PATCH 184/186] Install deps for daily workflow

Former-commit-id: 9b98c9edd3e8c9316514b7f9e0efc7a9f8d17851
---
 .github/workflows/daily.yml | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/daily.yml b/.github/workflows/daily.yml
index b6a9abb68..e03715d5c 100644
--- a/.github/workflows/daily.yml
+++ b/.github/workflows/daily.yml
@@ -11,7 +11,9 @@ jobs:
     steps:
     - uses: actions/checkout@v1
     - name: make
-      run: make
+      run: |
+          sudo apt-get -y install uuid-dev libcurl4-openssl-dev
+          make
     - name: test
       run: |
         sudo apt-get install tcl8.5
@@ -25,7 +27,9 @@ jobs:
     steps:
     - uses: actions/checkout@v1
     - name: make
-      run: make MALLOC=libc
+      run: |
+          sudo apt-get -y install uuid-dev libcurl4-openssl-dev
+          make MALLOC=libc
     - name: test
       run: |
         sudo apt-get install tcl8.5
@@ -39,7 +43,9 @@ jobs:
     steps:
     - uses: actions/checkout@v1
     - name: make
-      run: make valgrind
+      run: |
+          sudo apt-get -y install uuid-dev libcurl4-openssl-dev
+          make valgrind
     - name: test
       run: |
         sudo apt-get install tcl8.5 valgrind -y

From 975ca828d4bfc4c2d9c7bf9acd49adc75286b9f8 Mon Sep 17 00:00:00 2001
From: John Sully <john@csquare.ca>
Date: Sun, 24 May 2020 12:45:10 -0400
Subject: [PATCH 185/186] Quiet TSAN during exit

Former-commit-id: fd710d590a6297b74e06ce6832dd75bfca0a266f
---
 src/fastlock.cpp | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/fastlock.cpp b/src/fastlock.cpp
index bb8621906..cc6eea99a 100644
--- a/src/fastlock.cpp
+++ b/src/fastlock.cpp
@@ -194,8 +194,9 @@ static int futex(volatile unsigned *uaddr, int futex_op, int val,
 
 class DeadlockDetector
 {
+    fastlock m_lock { "deadlock detector" };    // destruct this first
     std::map<pid_t, fastlock *> m_mapwait;
-    fastlock m_lock { "deadlock detector" };
+
 public:
     void registerwait(fastlock *lock, pid_t thispid)
     {

From 0a1d2c99ac0b93d0058a402856e62c89cfb5d227 Mon Sep 17 00:00:00 2001
From: John Sully <john@csquare.ca>
Date: Sun, 24 May 2020 12:46:11 -0400
Subject: [PATCH 186/186] don't keep feeding data to clients pending close

Former-commit-id: 4db3f2a443a6f71d1486781005ee37546e89d6e1
---
 src/replication.cpp | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/replication.cpp b/src/replication.cpp
index 6b18d27ac..599c82030 100644
--- a/src/replication.cpp
+++ b/src/replication.cpp
@@ -1458,6 +1458,9 @@ void rdbPipeReadHandler(struct aeEventLoop *eventLoop, int fd, void *clientData,
 
             client *slave = (client*)connGetPrivateData(conn);
             std::unique_lock<fastlock> ul(slave->lock);
+            if(slave->flags.load(std::memory_order_relaxed) & CLIENT_CLOSE_ASAP)
+                continue;   // if we asked to free the client don't send any more data
+            
             // Normally it would be bug to talk a client conn from a different thread, but here we know nobody else will
             //  be sending anything while in this replication state so it is OK
             if ((nwritten = connWrite(conn, g_pserver->rdb_pipe_buff, g_pserver->rdb_pipe_bufflen)) == -1) {