From 63eb1114892070a82da04d2659f8985f4763c085 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Wed, 21 Feb 2018 11:04:13 +0200
Subject: [PATCH 01/60] Fix zrealloc to behave similarly to je_realloc when
 size is 0

According to C11, the behavior of realloc with size 0 is now deprecated.
it can either behave as free(ptr) and return NULL, or return a valid pointer.
but in zmalloc it can lead to zmalloc_oom_handler and panic.
and that can affect modules that use it.

It looks like both glibc allocator and jemalloc behave like so:
  realloc(malloc(32),0) returns NULL
  realloc(NULL,0) returns a valid pointer

This commit changes zmalloc to behave the same
---
 src/zmalloc.c | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/zmalloc.c b/src/zmalloc.c
index 094dd80fa..01ac8c797 100644
--- a/src/zmalloc.c
+++ b/src/zmalloc.c
@@ -147,6 +147,10 @@ void *zrealloc(void *ptr, size_t size) {
     size_t oldsize;
     void *newptr;
 
+    if (size == 0 && ptr!=NULL) {
+        zfree(ptr);
+        return NULL;
+    }
     if (ptr == NULL) return zmalloc(size);
 #ifdef HAVE_MALLOC_SIZE
     oldsize = zmalloc_size(ptr);

From aa123fff5a49f8d8c25b159c6b53b86b48ae7da5 Mon Sep 17 00:00:00 2001
From: Guy Benoish <guy.benoish@redislabs.com>
Date: Wed, 7 Mar 2018 10:40:37 +0700
Subject: [PATCH 02/60] Fix zlexrangespec mem-leak in genericZrangebylexCommand

---
 src/t_zset.c | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/t_zset.c b/src/t_zset.c
index f7f4c6eb2..fa7793b15 100644
--- a/src/t_zset.c
+++ b/src/t_zset.c
@@ -2856,7 +2856,10 @@ void genericZrangebylexCommand(client *c, int reverse) {
         while (remaining) {
             if (remaining >= 3 && !strcasecmp(c->argv[pos]->ptr,"limit")) {
                 if ((getLongFromObjectOrReply(c, c->argv[pos+1], &offset, NULL) != C_OK) ||
-                    (getLongFromObjectOrReply(c, c->argv[pos+2], &limit, NULL) != C_OK)) return;
+                    (getLongFromObjectOrReply(c, c->argv[pos+2], &limit, NULL) != C_OK)) {
+                    zslFreeLexRange(&range);
+                    return;
+                }
                 pos += 3; remaining -= 3;
             } else {
                 zslFreeLexRange(&range);

From 731fafa3d8f9c75eb0238a98361f8af6425983a5 Mon Sep 17 00:00:00 2001
From: Guy Benoish <guy.benoish@redislabs.com>
Date: Wed, 18 Apr 2018 13:01:53 +0300
Subject: [PATCH 03/60] Use memtoll() in 'CONFIG SET
 client-output-buffer-limit'

---
 src/config.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/config.c b/src/config.c
index a1122d059..a33981c63 100644
--- a/src/config.c
+++ b/src/config.c
@@ -983,8 +983,8 @@ void configSetCommand(client *c) {
             int soft_seconds;
 
             class = getClientTypeByName(v[j]);
-            hard = strtoll(v[j+1],NULL,10);
-            soft = strtoll(v[j+2],NULL,10);
+            hard = memtoll(v[j+1],NULL);
+            soft = memtoll(v[j+2],NULL);
             soft_seconds = strtoll(v[j+3],NULL,10);
 
             server.client_obuf_limits[class].hard_limit_bytes = hard;

From 327594b0307421dcfde4e6bc96029d43c5579b26 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Mon, 13 Aug 2018 17:43:29 +0300
Subject: [PATCH 04/60] Add log when server dies of SIGTERM during loading

this is very confusing to see the server disappears as if it got SIGKILL when it was not the case.
---
 src/server.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/server.c b/src/server.c
index b537ee04a..0c665033a 100644
--- a/src/server.c
+++ b/src/server.c
@@ -3780,6 +3780,7 @@ static void sigShutdownHandler(int sig) {
         rdbRemoveTempFile(getpid());
         exit(1); /* Exit with an error since this was not a clean shutdown. */
     } else if (server.loading) {
+        serverLogFromHandler(LL_WARNING, "Received shutdown signal during loading, exiting now.");
         exit(0);
     }
 

From 8e9c96adc2ebb996e181e065d6ab717d385bdca9 Mon Sep 17 00:00:00 2001
From: Guy Benoish <guy.benoish@redislabs.com>
Date: Thu, 13 Dec 2018 13:57:38 +0100
Subject: [PATCH 05/60] Check server.verbosity in RM_LogRaw

---
 src/module.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/module.c b/src/module.c
index 20d159d33..e553bc0d8 100644
--- a/src/module.c
+++ b/src/module.c
@@ -3421,6 +3421,8 @@ void RM_LogRaw(RedisModule *module, const char *levelstr, const char *fmt, va_li
     else if (!strcasecmp(levelstr,"warning")) level = LL_WARNING;
     else level = LL_VERBOSE; /* Default. */
 
+    if (level < server.verbosity) return;
+
     name_len = snprintf(msg, sizeof(msg),"<%s> ", module->name);
     vsnprintf(msg + name_len, sizeof(msg) - name_len, fmt, ap);
     serverLogRaw(level,msg);

From 2a907e7d0066f30b3dd04d3b6625799be1e1de90 Mon Sep 17 00:00:00 2001
From: MeirShpilraien <meir@redislabs.com>
Date: Sun, 9 Dec 2018 14:32:55 +0200
Subject: [PATCH 06/60] added module ability to register api to be used by
 other modules

---
 src/module.c      | 115 ++++++++++++++++++++++++++++++++++++++++++++++
 src/redismodule.h |   4 ++
 src/server.h      |   1 +
 3 files changed, 120 insertions(+)

diff --git a/src/module.c b/src/module.c
index 20d159d33..d177af24a 100644
--- a/src/module.c
+++ b/src/module.c
@@ -47,7 +47,16 @@ struct RedisModule {
     int ver;        /* Module version. We use just progressive integers. */
     int apiver;     /* Module API version as requested during initialization.*/
     list *types;    /* Module data types. */
+    list *usedBy;   /* list of modules names using this module api. */
+    list *using;    /* list of modules names that this module is using thier api . */
+    list *exportedFunctions;   /* list of function names exported by this module. */
 };
+
+struct ModuleExportedApi {
+    void* funcPointer;
+    struct RedisModule* module;
+};
+
 typedef struct RedisModule RedisModule;
 
 static dict *modules; /* Hash table of modules. SDS -> RedisModule ptr.*/
@@ -700,6 +709,9 @@ void RM_SetModuleAttribs(RedisModuleCtx *ctx, const char *name, int ver, int api
     module->ver = ver;
     module->apiver = apiver;
     module->types = listCreate();
+    module->usedBy = listCreate();
+    module->using = listCreate();
+    module->exportedFunctions = listCreate();
     ctx->module = module;
 }
 
@@ -4615,6 +4627,59 @@ void RM_GetRandomHexChars(char *dst, size_t len) {
     getRandomHexChars(dst,len);
 }
 
+/* Used to register an api to be used by other modules. */
+int RM_RegisterApi(RedisModuleCtx *ctx, const char *funcname, void *funcptr) {
+    struct ModuleExportedApi* eapi = zmalloc(sizeof(*eapi));
+    eapi->module = ctx->module;
+    eapi->funcPointer = funcptr;
+    if(dictAdd(server.exportedapi, (char*)funcname, eapi) != DICT_OK){
+        zfree(eapi);
+        return REDISMODULE_ERR;
+    }
+    listAddNodeHead(ctx->module->exportedFunctions, (char*)funcname);
+    return REDISMODULE_OK;
+}
+
+static inline int IsModuleInList(list *l, const char* moduleName){
+    listIter *iter = listGetIterator(l, AL_START_HEAD);
+    listNode *node = NULL;
+    while((node = listNext(iter))){
+        char* name = listNodeValue(node);
+        if(strcmp(name, moduleName) == 0){
+            listReleaseIterator(iter);
+            return 1;
+        }
+    }
+    listReleaseIterator(iter);
+    return 0;
+}
+
+static inline void RemoveFromList(list *l, const char* moduleName){
+    listIter *iter = listGetIterator(l, AL_START_HEAD);
+    listNode *node = NULL;
+    while((node = listNext(iter))){
+        char* name = listNodeValue(node);
+        if(strcmp(name, moduleName) == 0){
+            listDelNode(l, node);
+            return;
+        }
+    }
+    listReleaseIterator(iter);
+}
+
+void* RM_GetExportedApi(RedisModuleCtx *ctx, const char *funcname) {
+    dictEntry* entry = dictFind(server.exportedapi, funcname);
+    if(!entry){
+        return NULL;
+    }
+    struct ModuleExportedApi* eapi = dictGetVal(entry);
+    if(!IsModuleInList(eapi->module->usedBy, ctx->module->name)){
+        listAddNodeHead(eapi->module->usedBy, ctx->module->name);
+        listAddNodeHead(ctx->module->using, eapi->module->name);
+    }
+    return eapi->funcPointer;
+}
+
 /* --------------------------------------------------------------------------
  * Modules API internals
  * -------------------------------------------------------------------------- */
@@ -4735,6 +4800,28 @@ void moduleUnregisterCommands(struct RedisModule *module) {
     dictReleaseIterator(di);
 }
 
+void moduleUnregisterApi(struct RedisModule *module) {
+    listIter *iter = listGetIterator(module->exportedFunctions, AL_START_HEAD);
+    listNode* node = NULL;
+    while((node = listNext(iter))){
+        char* functionName = listNodeValue(node);
+        struct ModuleExportedApi* eapi = dictFetchValue(server.exportedapi, functionName);
+        serverAssert(eapi);
+        zfree(eapi);
+        dictDelete(server.exportedapi, functionName);
+    }
+    listReleaseIterator(iter);
+    iter = listGetIterator(module->using, AL_START_HEAD);
+    node = NULL;
+    while((node = listNext(iter))){
+        char* moduleName = listNodeValue(node);
+        struct RedisModule* usingModule = dictFetchValue(modules, moduleName);
+        serverAssert(usingModule);
+        RemoveFromList(usingModule->usedBy, module->name);
+    }
+    listReleaseIterator(iter);
+}
+
 /* Load a module and initialize it. On success C_OK is returned, otherwise
  * C_ERR is returned. */
 int moduleLoad(const char *path, void **module_argv, int module_argc) {
@@ -4794,6 +4881,12 @@ int moduleUnload(sds name) {
         return REDISMODULE_ERR;
     }
 
+    if (listLength(module->usedBy)) {
+        errno = EPERM;
+        return REDISMODULE_ERR;
+    }
+
+    moduleUnregisterApi(module);
     moduleUnregisterCommands(module);
 
     /* Remove any notification subscribers this module might have */
@@ -4826,6 +4919,7 @@ void moduleCommand(client *c) {
     if (c->argc == 2 && !strcasecmp(subcmd,"help")) {
         const char *help[] = {
 "LIST -- Return a list of loaded modules.",
+"LISTAPI <module-name> -- Return a list of exported api.",
 "LOAD <path> [arg ...] -- Load a module library from <path>.",
 "UNLOAD <name> -- Unload a module.",
 NULL
@@ -4858,6 +4952,9 @@ NULL
             case EBUSY:
                 errmsg = "the module exports one or more module-side data types, can't unload";
                 break;
+            case EPERM:
+                errmsg = "the module api is used by other modules, please unload them first and try again.";
+                break;
             default:
                 errmsg = "operation not possible.";
                 break;
@@ -4879,6 +4976,21 @@ NULL
             addReplyLongLong(c,module->ver);
         }
         dictReleaseIterator(di);
+    } else if (!strcasecmp(subcmd,"listapi") && c->argc == 3) {
+        char *moduleName = c->argv[2]->ptr;
+        struct RedisModule* module = dictFetchValue(modules, moduleName);
+        if(!module){
+            addReplyErrorFormat(c,"Error listing module api: no such module %s", moduleName);
+            return;
+        }
+        addReplyMultiBulkLen(c,listLength(module->exportedFunctions));
+        listIter *iter = listGetIterator(module->exportedFunctions, AL_START_HEAD);
+        listNode* node = NULL;
+        while((node = listNext(iter))){
+            char* functionName = listNodeValue(node);
+            addReplyBulkCString(c,functionName);
+        }
+        listReleaseIterator(iter);
     } else {
         addReplySubcommandSyntaxError(c);
         return;
@@ -4894,6 +5006,7 @@ size_t moduleCount(void) {
  * file so that's easy to seek it to add new entries. */
 void moduleRegisterCoreAPI(void) {
     server.moduleapi = dictCreate(&moduleAPIDictType,NULL);
+    server.exportedapi = dictCreate(&moduleAPIDictType,NULL);
     REGISTER_API(Alloc);
     REGISTER_API(Calloc);
     REGISTER_API(Realloc);
@@ -5044,4 +5157,6 @@ void moduleRegisterCoreAPI(void) {
     REGISTER_API(DictPrev);
     REGISTER_API(DictCompareC);
     REGISTER_API(DictCompare);
+    REGISTER_API(RegisterApi);
+    REGISTER_API(GetExportedApi);
 }
diff --git a/src/redismodule.h b/src/redismodule.h
index d18c38881..3c76fa02b 100644
--- a/src/redismodule.h
+++ b/src/redismodule.h
@@ -332,6 +332,8 @@ void REDISMODULE_API_FUNC(RedisModule_GetRandomBytes)(unsigned char *dst, size_t
 void REDISMODULE_API_FUNC(RedisModule_GetRandomHexChars)(char *dst, size_t len);
 void REDISMODULE_API_FUNC(RedisModule_SetDisconnectCallback)(RedisModuleBlockedClient *bc, RedisModuleDisconnectFunc callback);
 void REDISMODULE_API_FUNC(RedisModule_SetClusterFlags)(RedisModuleCtx *ctx, uint64_t flags);
+int REDISMODULE_API_FUNC(RedisModule_RegisterApi)(RedisModuleCtx *ctx, const char *funcname, void *funcptr);
+void* REDISMODULE_API_FUNC(RedisModule_GetExportedApi)(RedisModuleCtx *ctx, const char *funcname);
 #endif
 
 /* This is included inline inside each Redis module. */
@@ -492,6 +494,8 @@ static int RedisModule_Init(RedisModuleCtx *ctx, const char *name, int ver, int
     REDISMODULE_GET_API(GetRandomBytes);
     REDISMODULE_GET_API(GetRandomHexChars);
     REDISMODULE_GET_API(SetClusterFlags);
+    REDISMODULE_GET_API(RegisterApi);
+    REDISMODULE_GET_API(GetExportedApi);
 #endif
 
     if (RedisModule_IsModuleNameBusy && RedisModule_IsModuleNameBusy(name)) return REDISMODULE_ERR;
diff --git a/src/server.h b/src/server.h
index da4c6d45a..379cda058 100644
--- a/src/server.h
+++ b/src/server.h
@@ -955,6 +955,7 @@ struct redisServer {
     int always_show_logo;       /* Show logo even for non-stdout logging. */
     /* Modules */
     dict *moduleapi;            /* Exported APIs dictionary for modules. */
+    dict *exportedapi;          /* Api exported by other modules. */
     list *loadmodule_queue;     /* List of modules to load at startup. */
     int module_blocked_pipe[2]; /* Pipe used to awake the event loop if a
                                    client blocked on a module command needs

From ea2d2532381bc7ac7a20fb3af16b6f04b46b57b7 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Thu, 20 Dec 2018 17:56:38 +0100
Subject: [PATCH 07/60] Revert shared APIs to modify the design.

---
 src/module.c      | 115 ----------------------------------------------
 src/redismodule.h |   4 --
 src/server.h      |   1 -
 3 files changed, 120 deletions(-)

diff --git a/src/module.c b/src/module.c
index d177af24a..20d159d33 100644
--- a/src/module.c
+++ b/src/module.c
@@ -47,16 +47,7 @@ struct RedisModule {
     int ver;        /* Module version. We use just progressive integers. */
     int apiver;     /* Module API version as requested during initialization.*/
     list *types;    /* Module data types. */
-    list *usedBy;   /* list of modules names using this module api. */
-    list *using;    /* list of modules names that this module is using thier api . */
-    list *exportedFunctions;   /* list of function names exported by this module. */
 };
-
-struct ModuleExportedApi {
-    void* funcPointer;
-    struct RedisModule* module;
-};
-
 typedef struct RedisModule RedisModule;
 
 static dict *modules; /* Hash table of modules. SDS -> RedisModule ptr.*/
@@ -709,9 +700,6 @@ void RM_SetModuleAttribs(RedisModuleCtx *ctx, const char *name, int ver, int api
     module->ver = ver;
     module->apiver = apiver;
     module->types = listCreate();
-    module->usedBy = listCreate();
-    module->using = listCreate();
-    module->exportedFunctions = listCreate();
     ctx->module = module;
 }
 
@@ -4627,59 +4615,6 @@ void RM_GetRandomHexChars(char *dst, size_t len) {
     getRandomHexChars(dst,len);
 }
 
-/* Used to register an api to be used by other modules. */
-int RM_RegisterApi(RedisModuleCtx *ctx, const char *funcname, void *funcptr) {
-    struct ModuleExportedApi* eapi = zmalloc(sizeof(*eapi));
-    eapi->module = ctx->module;
-    eapi->funcPointer = funcptr;
-    if(dictAdd(server.exportedapi, (char*)funcname, eapi) != DICT_OK){
-        zfree(eapi);
-        return REDISMODULE_ERR;
-    }
-    listAddNodeHead(ctx->module->exportedFunctions, (char*)funcname);
-    return REDISMODULE_OK;
-}
-
-static inline int IsModuleInList(list *l, const char* moduleName){
-    listIter *iter = listGetIterator(l, AL_START_HEAD);
-    listNode *node = NULL;
-    while((node = listNext(iter))){
-        char* name = listNodeValue(node);
-        if(strcmp(name, moduleName) == 0){
-            listReleaseIterator(iter);
-            return 1;
-        }
-    }
-    listReleaseIterator(iter);
-    return 0;
-}
-
-static inline void RemoveFromList(list *l, const char* moduleName){
-    listIter *iter = listGetIterator(l, AL_START_HEAD);
-    listNode *node = NULL;
-    while((node = listNext(iter))){
-        char* name = listNodeValue(node);
-        if(strcmp(name, moduleName) == 0){
-            listDelNode(l, node);
-            return;
-        }
-    }
-    listReleaseIterator(iter);
-}
-
-void* RM_GetExportedApi(RedisModuleCtx *ctx, const char *funcname) {
-    dictEntry* entry = dictFind(server.exportedapi, funcname);
-    if(!entry){
-        return NULL;
-    }
-    struct ModuleExportedApi* eapi = dictGetVal(entry);
-    if(!IsModuleInList(eapi->module->usedBy, ctx->module->name)){
-        listAddNodeHead(eapi->module->usedBy, ctx->module->name);
-        listAddNodeHead(ctx->module->using, eapi->module->name);
-    }
-    return eapi->funcPointer;
-}
-
 /* --------------------------------------------------------------------------
  * Modules API internals
  * -------------------------------------------------------------------------- */
@@ -4800,28 +4735,6 @@ void moduleUnregisterCommands(struct RedisModule *module) {
     dictReleaseIterator(di);
 }
 
-void moduleUnregisterApi(struct RedisModule *module) {
-    listIter *iter = listGetIterator(module->exportedFunctions, AL_START_HEAD);
-    listNode* node = NULL;
-    while((node = listNext(iter))){
-        char* functionName = listNodeValue(node);
-        struct ModuleExportedApi* eapi = dictFetchValue(server.exportedapi, functionName);
-        serverAssert(eapi);
-        zfree(eapi);
-        dictDelete(server.exportedapi, functionName);
-    }
-    listReleaseIterator(iter);
-    iter = listGetIterator(module->using, AL_START_HEAD);
-    node = NULL;
-    while((node = listNext(iter))){
-        char* moduleName = listNodeValue(node);
-        struct RedisModule* usingModule = dictFetchValue(modules, moduleName);
-        serverAssert(usingModule);
-        RemoveFromList(usingModule->usedBy, module->name);
-    }
-    listReleaseIterator(iter);
-}
-
 /* Load a module and initialize it. On success C_OK is returned, otherwise
  * C_ERR is returned. */
 int moduleLoad(const char *path, void **module_argv, int module_argc) {
@@ -4881,12 +4794,6 @@ int moduleUnload(sds name) {
         return REDISMODULE_ERR;
     }
 
-    if (listLength(module->usedBy)) {
-        errno = EPERM;
-        return REDISMODULE_ERR;
-    }
-
-    moduleUnregisterApi(module);
     moduleUnregisterCommands(module);
 
     /* Remove any notification subscribers this module might have */
@@ -4919,7 +4826,6 @@ void moduleCommand(client *c) {
     if (c->argc == 2 && !strcasecmp(subcmd,"help")) {
         const char *help[] = {
 "LIST -- Return a list of loaded modules.",
-"LISTAPI <module-name> -- Return a list of exported api.",
 "LOAD <path> [arg ...] -- Load a module library from <path>.",
 "UNLOAD <name> -- Unload a module.",
 NULL
@@ -4952,9 +4858,6 @@ NULL
             case EBUSY:
                 errmsg = "the module exports one or more module-side data types, can't unload";
                 break;
-            case EPERM:
-                errmsg = "the module api is used by other modules, please unload them first and try again.";
-                break;
             default:
                 errmsg = "operation not possible.";
                 break;
@@ -4976,21 +4879,6 @@ NULL
             addReplyLongLong(c,module->ver);
         }
         dictReleaseIterator(di);
-    } else if (!strcasecmp(subcmd,"listapi") && c->argc == 3) {
-        char *moduleName = c->argv[2]->ptr;
-        struct RedisModule* module = dictFetchValue(modules, moduleName);
-        if(!module){
-            addReplyErrorFormat(c,"Error listing module api: no such module %s", moduleName);
-            return;
-        }
-        addReplyMultiBulkLen(c,listLength(module->exportedFunctions));
-        listIter *iter = listGetIterator(module->exportedFunctions, AL_START_HEAD);
-        listNode* node = NULL;
-        while((node = listNext(iter))){
-            char* functionName = listNodeValue(node);
-            addReplyBulkCString(c,functionName);
-        }
-        listReleaseIterator(iter);
     } else {
         addReplySubcommandSyntaxError(c);
         return;
@@ -5006,7 +4894,6 @@ size_t moduleCount(void) {
  * file so that's easy to seek it to add new entries. */
 void moduleRegisterCoreAPI(void) {
     server.moduleapi = dictCreate(&moduleAPIDictType,NULL);
-    server.exportedapi = dictCreate(&moduleAPIDictType,NULL);
     REGISTER_API(Alloc);
     REGISTER_API(Calloc);
     REGISTER_API(Realloc);
@@ -5157,6 +5044,4 @@ void moduleRegisterCoreAPI(void) {
     REGISTER_API(DictPrev);
     REGISTER_API(DictCompareC);
     REGISTER_API(DictCompare);
-    REGISTER_API(RegisterApi);
-    REGISTER_API(GetExportedApi);
 }
diff --git a/src/redismodule.h b/src/redismodule.h
index 3c76fa02b..d18c38881 100644
--- a/src/redismodule.h
+++ b/src/redismodule.h
@@ -332,8 +332,6 @@ void REDISMODULE_API_FUNC(RedisModule_GetRandomBytes)(unsigned char *dst, size_t
 void REDISMODULE_API_FUNC(RedisModule_GetRandomHexChars)(char *dst, size_t len);
 void REDISMODULE_API_FUNC(RedisModule_SetDisconnectCallback)(RedisModuleBlockedClient *bc, RedisModuleDisconnectFunc callback);
 void REDISMODULE_API_FUNC(RedisModule_SetClusterFlags)(RedisModuleCtx *ctx, uint64_t flags);
-int REDISMODULE_API_FUNC(RedisModule_RegisterApi)(RedisModuleCtx *ctx, const char *funcname, void *funcptr);
-void* REDISMODULE_API_FUNC(RedisModule_GetExportedApi)(RedisModuleCtx *ctx, const char *funcname);
 #endif
 
 /* This is included inline inside each Redis module. */
@@ -494,8 +492,6 @@ static int RedisModule_Init(RedisModuleCtx *ctx, const char *name, int ver, int
     REDISMODULE_GET_API(GetRandomBytes);
     REDISMODULE_GET_API(GetRandomHexChars);
     REDISMODULE_GET_API(SetClusterFlags);
-    REDISMODULE_GET_API(RegisterApi);
-    REDISMODULE_GET_API(GetExportedApi);
 #endif
 
     if (RedisModule_IsModuleNameBusy && RedisModule_IsModuleNameBusy(name)) return REDISMODULE_ERR;
diff --git a/src/server.h b/src/server.h
index 379cda058..da4c6d45a 100644
--- a/src/server.h
+++ b/src/server.h
@@ -955,7 +955,6 @@ struct redisServer {
     int always_show_logo;       /* Show logo even for non-stdout logging. */
     /* Modules */
     dict *moduleapi;            /* Exported APIs dictionary for modules. */
-    dict *exportedapi;          /* Api exported by other modules. */
     list *loadmodule_queue;     /* List of modules to load at startup. */
     int module_blocked_pipe[2]; /* Pipe used to awake the event loop if a
                                    client blocked on a module command needs

From f7e99b07f36c8989ca704295318abd50fde4f690 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Thu, 20 Dec 2018 12:06:24 +0100
Subject: [PATCH 08/60] Modules shared API: initial core functions.

Based on ideas and code in PR #5560 by @MeirShpilraien.
---
 src/module.c | 86 ++++++++++++++++++++++++++++++++++++++++++++++++++++
 src/server.h |  4 ++-
 2 files changed, 89 insertions(+), 1 deletion(-)

diff --git a/src/module.c b/src/module.c
index 20d159d33..2914b5903 100644
--- a/src/module.c
+++ b/src/module.c
@@ -47,9 +47,21 @@ struct RedisModule {
     int ver;        /* Module version. We use just progressive integers. */
     int apiver;     /* Module API version as requested during initialization.*/
     list *types;    /* Module data types. */
+    list *usedby;   /* List of modules using APIs from this one. */
+    list *using;    /* List of modules we use some APIs of. */
 };
 typedef struct RedisModule RedisModule;
 
+/* This represents a shared API. Shared APIs will be used to populate
+ * the server.sharedapi dictionary, mapping names of APIs exported by
+ * modules for other modules to use, to their structure specifying the
+ * function pointer that can be called. */
+struct RedisModuleSharedAPI {
+    void *func;
+    RedisModule *module;
+};
+typedef struct RedisModuleSharedAPI RedisModuleSharedAPI;
+
 static dict *modules; /* Hash table of modules. SDS -> RedisModule ptr.*/
 
 /* Entries in the context->amqueue array, representing objects to free
@@ -700,6 +712,8 @@ void RM_SetModuleAttribs(RedisModuleCtx *ctx, const char *name, int ver, int api
     module->ver = ver;
     module->apiver = apiver;
     module->types = listCreate();
+    module->usedby = listCreate();
+    module->using = listCreate();
     ctx->module = module;
 }
 
@@ -4615,6 +4629,77 @@ void RM_GetRandomHexChars(char *dst, size_t len) {
     getRandomHexChars(dst,len);
 }
 
+/* --------------------------------------------------------------------------
+ * Modules API exporting / importing
+ * -------------------------------------------------------------------------- */
+
+/* This function is called by a module in order to export some API with a
+ * given name. Other modules will be able to use this API by calling the
+ * symmetrical function RM_GetSharedAPI() and casting the return value to
+ * the right function pointer.
+ *
+ * The function will return REDISMODULE_OK if the name is not already taken,
+ * otherwise REDISMODULE_ERR will be returned and no operation will be
+ * performed.
+ *
+ * IMPORTANT: the apiname argument should be a string literal with static
+ * lifetime. The API relies on the fact that it will always be valid in
+ * the future. */
+int RM_ExportSharedAPI(RedisModuleCtx *ctx, const char *apiname, void *func) {
+    RedisModuleSharedAPI *sapi = zmalloc(sizeof(*sapi));
+    sapi->module = ctx->module;
+    sapi->func = func;
+    if (dictAdd(server.sharedapi, (char*)apiname, sapi) != DICT_OK) {
+        zfree(sapi);
+        return REDISMODULE_ERR;
+    }
+    return REDISMODULE_OK;
+}
+
+/* Request an exported API pointer. The return value is just a void pointer
+ * that the caller of this function will be required to cast to the right
+ * function pointer, so this is a private contract between modules.
+ *
+ * If the requested API is not available then NULL is returned. Because
+ * modules can be loaded at different times with different order, this
+ * function calls should be put inside some module generic API registering
+ * step, that is called every time a module attempts to execute a
+ * command that requires external APIs: if some API cannot be resolved, the
+ * command should return an error.
+ *
+ * Here is an exmaple:
+ *
+ *     int ... myCommandImplementation() {
+ *        if (getExternalAPIs() == 0) {
+ *             reply with an error here if we cannot have the APIs
+ *        }
+ *        // Use the API:
+ *        myFunctionPointer(foo);
+ *     }
+ *
+ * And the function registerAPI() is:
+ *
+ *     int getExternalAPIs(void) {
+ *         static int api_loaded = 0;
+ *         if (api_loaded != 0) return 1; // APIs already resolved.
+ *
+ *         myFunctionPointer = RedisModule_GetOtherModuleAPI("...");
+ *         if (myFunctionPointer == NULL) return 0;
+ *
+ *         return 1;
+ *     }
+ */
+void *RM_GetSharedAPI(RedisModuleCtx *ctx, const char *apiname) {
+    dictEntry *de = dictFind(server.sharedapi, apiname);
+    if (de == NULL) return NULL;
+    RedisModuleSharedAPI *sapi = dictGetVal(de);
+    if (listSearchKey(sapi->module->usedby,ctx->module) == NULL) {
+        listAddNodeTail(sapi->module->usedby,ctx->module);
+        listAddNodeTail(ctx->module->using,sapi->module);
+    }
+    return sapi->func;
+}
+
 /* --------------------------------------------------------------------------
  * Modules API internals
  * -------------------------------------------------------------------------- */
@@ -4894,6 +4979,7 @@ size_t moduleCount(void) {
  * file so that's easy to seek it to add new entries. */
 void moduleRegisterCoreAPI(void) {
     server.moduleapi = dictCreate(&moduleAPIDictType,NULL);
+    server.sharedapi = dictCreate(&moduleAPIDictType,NULL);
     REGISTER_API(Alloc);
     REGISTER_API(Calloc);
     REGISTER_API(Realloc);
diff --git a/src/server.h b/src/server.h
index da4c6d45a..3c2ecdd23 100644
--- a/src/server.h
+++ b/src/server.h
@@ -954,7 +954,9 @@ struct redisServer {
     size_t initial_memory_usage; /* Bytes used after initialization. */
     int always_show_logo;       /* Show logo even for non-stdout logging. */
     /* Modules */
-    dict *moduleapi;            /* Exported APIs dictionary for modules. */
+    dict *moduleapi;            /* Exported core APIs dictionary for modules. */
+    dict *sharedapi;            /* Like moduleapi but containing the APIs that
+                                   modules share with each other. */
     list *loadmodule_queue;     /* List of modules to load at startup. */
     int module_blocked_pipe[2]; /* Pipe used to awake the event loop if a
                                    client blocked on a module command needs

From a41459ea5b6d8f13da8713ccf49746130f0fd2c8 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Thu, 20 Dec 2018 17:16:39 +0100
Subject: [PATCH 09/60] Modules shared API: unregister APIs function.

---
 src/module.c | 25 +++++++++++++++++++++++++
 1 file changed, 25 insertions(+)

diff --git a/src/module.c b/src/module.c
index 2914b5903..eafdd81f6 100644
--- a/src/module.c
+++ b/src/module.c
@@ -4700,6 +4700,29 @@ void *RM_GetSharedAPI(RedisModuleCtx *ctx, const char *apiname) {
     return sapi->func;
 }
 
+/* Remove all the APIs registered by the specified module. Usually you
+ * want this when the module is going to be unloaded. This function
+ * assumes that's caller responsibility to make sure the APIs are not
+ * used by other modules.
+ *
+ * The number of unregistered APIs is returned. */
+int moduleUnregisterSharedAPI(RedisModule *module) {
+    int count = 0;
+    dictIterator *di = dictGetSafeIterator(server.sharedapi);
+    dictEntry *de;
+    while ((de = dictNext(di)) != NULL) {
+        const char *apiname = dictGetKey(de);
+        RedisModuleSharedAPI *sapi = dictGetVal(de);
+        if (sapi->module == module) {
+            dictDelete(server.sharedapi,apiname);
+            zfree(sapi);
+            count++;
+        }
+    }
+    dictReleaseIterator(di);
+    return count;
+}
+
 /* --------------------------------------------------------------------------
  * Modules API internals
  * -------------------------------------------------------------------------- */
@@ -4843,6 +4866,7 @@ int moduleLoad(const char *path, void **module_argv, int module_argc) {
     if (onload((void*)&ctx,module_argv,module_argc) == REDISMODULE_ERR) {
         if (ctx.module) {
             moduleUnregisterCommands(ctx.module);
+            moduleUnregisterSharedAPI(ctx.module);
             moduleFreeModuleStructure(ctx.module);
         }
         dlclose(handle);
@@ -4880,6 +4904,7 @@ int moduleUnload(sds name) {
     }
 
     moduleUnregisterCommands(module);
+    moduleUnregisterSharedAPI(module);
 
     /* Remove any notification subscribers this module might have */
     moduleUnsubscribeNotifications(module);

From f25bcefb825b3bd767bb8aa9a781bcf4bfe2aec5 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Thu, 20 Dec 2018 17:31:55 +0100
Subject: [PATCH 10/60] Modules shared API: prevent unloading of used modules.

---
 src/module.c | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/src/module.c b/src/module.c
index eafdd81f6..4f0e5b126 100644
--- a/src/module.c
+++ b/src/module.c
@@ -4896,11 +4896,12 @@ int moduleUnload(sds name) {
     if (module == NULL) {
         errno = ENOENT;
         return REDISMODULE_ERR;
-    }
-
-    if (listLength(module->types)) {
+    } else if (listLength(module->types)) {
         errno = EBUSY;
         return REDISMODULE_ERR;
+    } else if (listLength(module->usedby)) {
+        errno = EPERM;
+        return REDISMODULE_ERR;
     }
 
     moduleUnregisterCommands(module);
@@ -4966,7 +4967,12 @@ NULL
                 errmsg = "no such module with that name";
                 break;
             case EBUSY:
-                errmsg = "the module exports one or more module-side data types, can't unload";
+                errmsg = "the module exports one or more module-side data "
+                         "types, can't unload";
+                break;
+            case EPERM:
+                errmsg = "the module exports APIs used by other modules. "
+                         "Please unload them first and try again";
                 break;
             default:
                 errmsg = "operation not possible.";

From 692f2297b90f9511e4b71e699157c1e737f026fe Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Thu, 20 Dec 2018 17:40:55 +0100
Subject: [PATCH 11/60] Modules shared API: also unregister the module as user.

---
 src/module.c | 23 +++++++++++++++++++++++
 1 file changed, 23 insertions(+)

diff --git a/src/module.c b/src/module.c
index 4f0e5b126..7bb146954 100644
--- a/src/module.c
+++ b/src/module.c
@@ -4723,6 +4723,27 @@ int moduleUnregisterSharedAPI(RedisModule *module) {
     return count;
 }
 
+/* Remove the specified module as an user of APIs of ever other module.
+ * This is usually called when a module is unloaded.
+ *
+ * Returns the number of modules this module was using APIs from. */
+int moduleUnregisterUsedAPI(RedisModule *module) {
+    listIter li;
+    listNode *ln;
+    int count = 0;
+
+    listRewind(module->using,&li);
+    while((ln = listNext(&li))) {
+        RedisModule *used = ln->value;
+        listNode *ln = listSearchKey(used->usedby,module);
+        if (ln) {
+            listDelNode(module->using,ln);
+            count++;
+        }
+    }
+    return count;
+}
+
 /* --------------------------------------------------------------------------
  * Modules API internals
  * -------------------------------------------------------------------------- */
@@ -4867,6 +4888,7 @@ int moduleLoad(const char *path, void **module_argv, int module_argc) {
         if (ctx.module) {
             moduleUnregisterCommands(ctx.module);
             moduleUnregisterSharedAPI(ctx.module);
+            moduleUnregisterUsedAPI(ctx.module);
             moduleFreeModuleStructure(ctx.module);
         }
         dlclose(handle);
@@ -4906,6 +4928,7 @@ int moduleUnload(sds name) {
 
     moduleUnregisterCommands(module);
     moduleUnregisterSharedAPI(module);
+    moduleUnregisterUsedAPI(module);
 
     /* Remove any notification subscribers this module might have */
     moduleUnsubscribeNotifications(module);

From b91f7656fd8b640b8bbde2b65b65354ebc10f9dc Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Thu, 20 Dec 2018 17:44:51 +0100
Subject: [PATCH 12/60] Modules shared API: export new core APIs.

---
 src/module.c      | 2 ++
 src/redismodule.h | 4 ++++
 2 files changed, 6 insertions(+)

diff --git a/src/module.c b/src/module.c
index 7bb146954..f2582193c 100644
--- a/src/module.c
+++ b/src/module.c
@@ -5184,4 +5184,6 @@ void moduleRegisterCoreAPI(void) {
     REGISTER_API(DictPrev);
     REGISTER_API(DictCompareC);
     REGISTER_API(DictCompare);
+    REGISTER_API(ExportSharedAPI);
+    REGISTER_API(GetSharedAPI);
 }
diff --git a/src/redismodule.h b/src/redismodule.h
index d18c38881..3a7da18fe 100644
--- a/src/redismodule.h
+++ b/src/redismodule.h
@@ -332,6 +332,8 @@ void REDISMODULE_API_FUNC(RedisModule_GetRandomBytes)(unsigned char *dst, size_t
 void REDISMODULE_API_FUNC(RedisModule_GetRandomHexChars)(char *dst, size_t len);
 void REDISMODULE_API_FUNC(RedisModule_SetDisconnectCallback)(RedisModuleBlockedClient *bc, RedisModuleDisconnectFunc callback);
 void REDISMODULE_API_FUNC(RedisModule_SetClusterFlags)(RedisModuleCtx *ctx, uint64_t flags);
+int REDISMODULE_API_FUNC(RedisModule_ExportSharedAPI)(RedisModuleCtx *ctx, const char *apiname, void *func);
+void *REDISMODULE_API_FUNC(RedisModule_GetSharedAPI)(RedisModuleCtx *ctx, const char *apiname);
 #endif
 
 /* This is included inline inside each Redis module. */
@@ -492,6 +494,8 @@ static int RedisModule_Init(RedisModuleCtx *ctx, const char *name, int ver, int
     REDISMODULE_GET_API(GetRandomBytes);
     REDISMODULE_GET_API(GetRandomHexChars);
     REDISMODULE_GET_API(SetClusterFlags);
+    REDISMODULE_GET_API(ExportSharedAPI);
+    REDISMODULE_GET_API(GetSharedAPI);
 #endif
 
     if (RedisModule_IsModuleNameBusy && RedisModule_IsModuleNameBusy(name)) return REDISMODULE_ERR;

From 59b90db74592a7e07e03368004f679bcf562e0d5 Mon Sep 17 00:00:00 2001
From: Guy Benoish <guy.benoish@redislabs.com>
Date: Wed, 23 Jan 2019 11:11:57 +0200
Subject: [PATCH 13/60] ZPOP should return an empty array if COUNT=0

---
 src/t_zset.c | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/t_zset.c b/src/t_zset.c
index 0427ee887..1c3da1a28 100644
--- a/src/t_zset.c
+++ b/src/t_zset.c
@@ -3140,7 +3140,10 @@ void genericZpopCommand(client *c, robj **keyv, int keyc, int where, int emitkey
     if (countarg) {
         if (getLongFromObjectOrReply(c,countarg,&count,NULL) != C_OK)
             return;
-        if (count < 0) count = 1;
+        if (count <= 0) {
+            addReplyNullArray(c);
+            return;
+        }
     }
 
     /* Check type and break on the first error, otherwise identify candidate. */

From 7050bf5e2ecdb9d037e4f7a8a9c4ce415afa4157 Mon Sep 17 00:00:00 2001
From: Guy Benoish <guy.benoish@redislabs.com>
Date: Mon, 28 Jan 2019 17:58:11 +0200
Subject: [PATCH 14/60] Increase string2ld's buffer size (and fix HINCRBYFLOAT)

The string representation of `long double` may take
up to ~5000 chars (see PR #3745).

Before this fix HINCRBYFLOAT would never overflow (since
the string could not exceed 256 chars). Now it can.
---
 src/t_hash.c | 4 ++++
 src/util.c   | 2 +-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/t_hash.c b/src/t_hash.c
index d8aee6572..bc70e4051 100644
--- a/src/t_hash.c
+++ b/src/t_hash.c
@@ -615,6 +615,10 @@ void hincrbyfloatCommand(client *c) {
     }
 
     value += incr;
+    if (isnan(value) || isinf(value)) {
+        addReplyError(c,"increment would produce NaN or Infinity");
+        return;
+    }
 
     char buf[MAX_LONG_DOUBLE_CHARS];
     int len = ld2string(buf,sizeof(buf),value,1);
diff --git a/src/util.c b/src/util.c
index 66d599190..783bcf83b 100644
--- a/src/util.c
+++ b/src/util.c
@@ -447,7 +447,7 @@ int string2l(const char *s, size_t slen, long *lval) {
  * a double: no spaces or other characters before or after the string
  * representing the number are accepted. */
 int string2ld(const char *s, size_t slen, long double *dp) {
-    char buf[256];
+    char buf[MAX_LONG_DOUBLE_CHARS];
     long double value;
     char *eptr;
 

From 9cd3b12cdff082e5499d72a7ece3eafee502657b Mon Sep 17 00:00:00 2001
From: Guy Benoish <guy.benoish@redislabs.com>
Date: Tue, 12 Feb 2019 14:21:21 +0100
Subject: [PATCH 15/60] Trim SDS free space of retained module strings

In some cases processMultibulkBuffer uses sdsMakeRoomFor to
expand the querybuf, but later in some cases it uses that query
buffer as is for an argv element (see "Optimization"), which means
that the sds in argv may have a lot of wasted space, and then in case
modules keep that argv RedisString inside their data structure, this
space waste will remain for long (until restarted from rdb).
---
 src/module.c | 11 +++++++++++
 src/object.c | 17 ++++++++++++-----
 src/sds.c    |  4 ++++
 src/server.h |  1 +
 4 files changed, 28 insertions(+), 5 deletions(-)

diff --git a/src/module.c b/src/module.c
index 81982ba76..5d3451916 100644
--- a/src/module.c
+++ b/src/module.c
@@ -509,6 +509,17 @@ void RedisModuleCommandDispatcher(client *c) {
     cp->func(&ctx,(void**)c->argv,c->argc);
     moduleHandlePropagationAfterCommandCallback(&ctx);
     moduleFreeContext(&ctx);
+
+    /* In some cases processMultibulkBuffer uses sdsMakeRoomFor to
+     * expand the querybuf, but later in some cases it uses that query
+     * buffer as is for an argv element (see "Optimization"), which means
+     * that the sds in argv may have a lot of wasted space, and then in case
+     * modules keep that argv RedisString inside their data structure, this
+     * space waste will remain for long (until restarted from rdb). */
+    for (int i = 0; i < c->argc; i++) {
+        if (c->argv[i]->refcount > 1)
+            trimStringObjectIfNeeded(c->argv[i]);
+    }
 }
 
 /* This function returns the list of keys, with the same interface as the
diff --git a/src/object.c b/src/object.c
index ec0bd02ee..42c247b87 100644
--- a/src/object.c
+++ b/src/object.c
@@ -415,6 +415,17 @@ int isObjectRepresentableAsLongLong(robj *o, long long *llval) {
     }
 }
 
+void trimStringObjectIfNeeded(robj *o) {
+    /* Optimize the SDS string inside the string object to require
+     * little space, in case there is more than 10% of free space
+     * at the end of the SDS string. */
+    if (o->encoding == OBJ_ENCODING_RAW &&
+        sdsavail(o->ptr) > sdslen(o->ptr)/10)
+    {
+        o->ptr = sdsRemoveFreeSpace(o->ptr);
+    }
+}
+
 /* Try to encode a string object in order to save space */
 robj *tryObjectEncoding(robj *o) {
     long value;
@@ -484,11 +495,7 @@ robj *tryObjectEncoding(robj *o) {
      * We do that only for relatively large strings as this branch
      * is only entered if the length of the string is greater than
      * OBJ_ENCODING_EMBSTR_SIZE_LIMIT. */
-    if (o->encoding == OBJ_ENCODING_RAW &&
-        sdsavail(s) > len/10)
-    {
-        o->ptr = sdsRemoveFreeSpace(o->ptr);
-    }
+    trimStringObjectIfNeeded(o);
 
     /* Return the original object. */
     return o;
diff --git a/src/sds.c b/src/sds.c
index 330c955e8..cd60946bd 100644
--- a/src/sds.c
+++ b/src/sds.c
@@ -257,8 +257,12 @@ sds sdsRemoveFreeSpace(sds s) {
     char type, oldtype = s[-1] & SDS_TYPE_MASK;
     int hdrlen, oldhdrlen = sdsHdrSize(oldtype);
     size_t len = sdslen(s);
+    size_t avail = sdsavail(s);
     sh = (char*)s-oldhdrlen;
 
+    /* Return ASAP if there is no space left. */
+    if (avail == 0) return s;
+
     /* Check what would be the minimum SDS header that is just good enough to
      * fit this string. */
     type = sdsReqType(len);
diff --git a/src/server.h b/src/server.h
index a396e1cf7..34b4cd8d5 100644
--- a/src/server.h
+++ b/src/server.h
@@ -1656,6 +1656,7 @@ int compareStringObjects(robj *a, robj *b);
 int collateStringObjects(robj *a, robj *b);
 int equalStringObjects(robj *a, robj *b);
 unsigned long long estimateObjectIdleTime(robj *o);
+void trimStringObjectIfNeeded(robj *o);
 #define sdsEncodedObject(objptr) (objptr->encoding == OBJ_ENCODING_RAW || objptr->encoding == OBJ_ENCODING_EMBSTR)
 
 /* Synchronous I/O with timeout */

From 24300677e7172c9081a865edbd417a89b7987042 Mon Sep 17 00:00:00 2001
From: Guy Benoish <guy.benoish@redislabs.com>
Date: Thu, 14 Mar 2019 12:11:16 +0100
Subject: [PATCH 16/60] Fix mismatching keyspace notification classes

---
 src/geo.c    | 2 +-
 src/t_list.c | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/geo.c b/src/geo.c
index 91a0421f5..826d11ff5 100644
--- a/src/geo.c
+++ b/src/geo.c
@@ -659,7 +659,7 @@ void georadiusGeneric(client *c, int flags) {
             zsetConvertToZiplistIfNeeded(zobj,maxelelen);
             setKey(c->db,storekey,zobj);
             decrRefCount(zobj);
-            notifyKeyspaceEvent(NOTIFY_LIST,"georadiusstore",storekey,
+            notifyKeyspaceEvent(NOTIFY_ZSET,"georadiusstore",storekey,
                                 c->db->id);
             server.dirty += returned_items;
         } else if (dbDelete(c->db,storekey)) {
diff --git a/src/t_list.c b/src/t_list.c
index 451ffb4b5..45d2e3317 100644
--- a/src/t_list.c
+++ b/src/t_list.c
@@ -520,7 +520,7 @@ void lremCommand(client *c) {
 
     if (removed) {
         signalModifiedKey(c->db,c->argv[1]);
-        notifyKeyspaceEvent(NOTIFY_GENERIC,"lrem",c->argv[1],c->db->id);
+        notifyKeyspaceEvent(NOTIFY_LIST,"lrem",c->argv[1],c->db->id);
     }
 
     if (listTypeLength(subject) == 0) {

From 9c504573f18ac6ad15360daa3d0b44179e693ccb Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Thu, 14 Mar 2019 12:47:36 +0100
Subject: [PATCH 17/60] Improve comments after merging #5834.

---
 src/module.c | 15 ++++++++++-----
 src/object.c |  7 ++++---
 2 files changed, 14 insertions(+), 8 deletions(-)

diff --git a/src/module.c b/src/module.c
index 28a4d3e64..e69d3dc61 100644
--- a/src/module.c
+++ b/src/module.c
@@ -523,12 +523,17 @@ void RedisModuleCommandDispatcher(client *c) {
     moduleFreeContext(&ctx);
 
     /* In some cases processMultibulkBuffer uses sdsMakeRoomFor to
-     * expand the querybuf, but later in some cases it uses that query
-     * buffer as is for an argv element (see "Optimization"), which means
-     * that the sds in argv may have a lot of wasted space, and then in case
-     * modules keep that argv RedisString inside their data structure, this
-     * space waste will remain for long (until restarted from rdb). */
+     * expand the query buffer, and in order to avoid a big object copy
+     * the query buffer SDS may be used directly as the SDS string backing
+     * the client argument vectors: sometimes this will result in the SDS
+     * string having unused space at the end. Later if a module takes ownership
+     * of the RedisString, such space will be wasted forever. Inside the
+     * Redis core this is not a problem because tryObjectEncoding() is called
+     * before storing strings in the key space. Here we need to do it
+     * for the module. */
     for (int i = 0; i < c->argc; i++) {
+        /* Only do the work if the module took ownership of the object:
+         * in that case the refcount is no longer 1. */
         if (c->argv[i]->refcount > 1)
             trimStringObjectIfNeeded(c->argv[i]);
     }
diff --git a/src/object.c b/src/object.c
index 42c247b87..24e99d191 100644
--- a/src/object.c
+++ b/src/object.c
@@ -415,10 +415,11 @@ int isObjectRepresentableAsLongLong(robj *o, long long *llval) {
     }
 }
 
+/* Optimize the SDS string inside the string object to require little space,
+ * in case there is more than 10% of free space at the end of the SDS
+ * string. This happens because SDS strings tend to overallocate to avoid
+ * wasting too much time in allocations when appending to the string. */
 void trimStringObjectIfNeeded(robj *o) {
-    /* Optimize the SDS string inside the string object to require
-     * little space, in case there is more than 10% of free space
-     * at the end of the SDS string. */
     if (o->encoding == OBJ_ENCODING_RAW &&
         sdsavail(o->ptr) > sdslen(o->ptr)/10)
     {

From e8568bb34189cf45b3816a87c7c24c996e1cd827 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Thu, 14 Mar 2019 17:06:59 +0100
Subject: [PATCH 18/60] Fix objectSetLRUOrLFU() when LFU underflows.

---
 src/object.c | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/src/object.c b/src/object.c
index 24e99d191..234e11f8a 100644
--- a/src/object.c
+++ b/src/object.c
@@ -1199,7 +1199,7 @@ sds getMemoryDoctorReport(void) {
 
 /* Set the object LRU/LFU depending on server.maxmemory_policy.
  * The lfu_freq arg is only relevant if policy is MAXMEMORY_FLAG_LFU.
- * The lru_idle and lru_clock args are only relevant if policy 
+ * The lru_idle and lru_clock args are only relevant if policy
  * is MAXMEMORY_FLAG_LRU.
  * Either or both of them may be <0, in that case, nothing is set. */
 void objectSetLRUOrLFU(robj *val, long long lfu_freq, long long lru_idle,
@@ -1210,16 +1210,20 @@ void objectSetLRUOrLFU(robj *val, long long lfu_freq, long long lru_idle,
             val->lru = (LFUGetTimeInMinutes()<<8) | lfu_freq;
         }
     } else if (lru_idle >= 0) {
-        /* Serialized LRU idle time is in seconds. Scale
+        /* Provided LRU idle time is in seconds. Scale
          * according to the LRU clock resolution this Redis
          * instance was compiled with (normally 1000 ms, so the
          * below statement will expand to lru_idle*1000/1000. */
         lru_idle = lru_idle*1000/LRU_CLOCK_RESOLUTION;
-        val->lru = lru_clock - lru_idle;
-        /* If the lru field overflows (since LRU it is a wrapping
-         * clock), the best we can do is to provide the maximum
-         * representable idle time. */
-        if (val->lru < 0) val->lru = lru_clock+1;
+        long lru_abs = lru_clock - lru_idle; /* Absolute access time. */
+        /* If the LRU field underflows (since LRU it is a wrapping
+         * clock), the best we can do is to provide a large enough LRU
+         * that is half-way in the circlular LRU clock we use: this way
+         * the computed idle time for this object will stay high for quite
+         * some time. */
+        if (lru_abs < 0)
+            lru_abs = (lru_clock+(LRU_CLOCK_MAX/2)) % LRU_CLOCK_MAX;
+        val->lru = lru_abs;
     }
 }
 

From 5199f1dc584b72c2c6c7afab19cca6bfab71c5d3 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Thu, 14 Mar 2019 17:51:14 +0100
Subject: [PATCH 19/60] Fix ZPOP return type when COUNT=0. Related to #5799.

---
 src/t_zset.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/t_zset.c b/src/t_zset.c
index 0daa6d643..fb7078abd 100644
--- a/src/t_zset.c
+++ b/src/t_zset.c
@@ -3144,7 +3144,7 @@ void genericZpopCommand(client *c, robj **keyv, int keyc, int where, int emitkey
         if (getLongFromObjectOrReply(c,countarg,&count,NULL) != C_OK)
             return;
         if (count <= 0) {
-            addReplyNullArray(c);
+            addReply(c,shared.emptyarray);
             return;
         }
     }

From dd405d4026274d139216deef695f99df91ee69bd Mon Sep 17 00:00:00 2001
From: Yossi Gottlieb <yossigo@gmail.com>
Date: Wed, 30 Nov 2016 21:47:02 +0200
Subject: [PATCH 20/60] Add RedisModule_GetKeyNameFromIO().

---
 src/aof.c             |  2 +-
 src/cluster.c         | 10 +++++-----
 src/module.c          |  9 +++++++++
 src/rdb.c             | 14 +++++++-------
 src/rdb.h             |  4 ++--
 src/redis-check-rdb.c |  2 +-
 src/redismodule.h     |  2 ++
 src/server.h          |  4 +++-
 8 files changed, 30 insertions(+), 17 deletions(-)

diff --git a/src/aof.c b/src/aof.c
index cafcf961c..615eebd01 100644
--- a/src/aof.c
+++ b/src/aof.c
@@ -1239,7 +1239,7 @@ int rewriteModuleObject(rio *r, robj *key, robj *o) {
     RedisModuleIO io;
     moduleValue *mv = o->ptr;
     moduleType *mt = mv->type;
-    moduleInitIOContext(io,mt,r);
+    moduleInitIOContext(io,mt,r,key);
     mt->aof_rewrite(&io,key,mv->value);
     if (io.ctx) {
         moduleFreeContext(io.ctx);
diff --git a/src/cluster.c b/src/cluster.c
index 50a9ae687..c85e3791d 100644
--- a/src/cluster.c
+++ b/src/cluster.c
@@ -4776,7 +4776,7 @@ NULL
 
 /* Generates a DUMP-format representation of the object 'o', adding it to the
  * io stream pointed by 'rio'. This function can't fail. */
-void createDumpPayload(rio *payload, robj *o) {
+void createDumpPayload(rio *payload, robj *o, robj *key) {
     unsigned char buf[2];
     uint64_t crc;
 
@@ -4784,7 +4784,7 @@ void createDumpPayload(rio *payload, robj *o) {
      * byte followed by the serialized object. This is understood by RESTORE. */
     rioInitWithBuffer(payload,sdsempty());
     serverAssert(rdbSaveObjectType(payload,o));
-    serverAssert(rdbSaveObject(payload,o));
+    serverAssert(rdbSaveObject(payload,o,key));
 
     /* Write the footer, this is how it looks like:
      * ----------------+---------------------+---------------+
@@ -4842,7 +4842,7 @@ void dumpCommand(client *c) {
     }
 
     /* Create the DUMP encoded representation. */
-    createDumpPayload(&payload,o);
+    createDumpPayload(&payload,o,c->argv[1]);
 
     /* Transfer to the client */
     dumpobj = createObject(OBJ_STRING,payload.io.buffer.ptr);
@@ -4915,7 +4915,7 @@ void restoreCommand(client *c) {
 
     rioInitWithBuffer(&payload,c->argv[3]->ptr);
     if (((type = rdbLoadObjectType(&payload)) == -1) ||
-        ((obj = rdbLoadObject(type,&payload)) == NULL))
+        ((obj = rdbLoadObject(type,&payload,c->argv[1])) == NULL))
     {
         addReplyError(c,"Bad data format");
         return;
@@ -5203,7 +5203,7 @@ try_again:
 
         /* Emit the payload argument, that is the serialized object using
          * the DUMP format. */
-        createDumpPayload(&payload,ov[j]);
+        createDumpPayload(&payload,ov[j],kv[j]);
         serverAssertWithInfo(c,NULL,
             rioWriteBulkString(&cmd,payload.io.buffer.ptr,
                                sdslen(payload.io.buffer.ptr)));
diff --git a/src/module.c b/src/module.c
index e69d3dc61..e1ffd7313 100644
--- a/src/module.c
+++ b/src/module.c
@@ -3438,6 +3438,14 @@ RedisModuleCtx *RM_GetContextFromIO(RedisModuleIO *io) {
     return io->ctx;
 }
 
+/* Returns a RedisModuleString with the name of the key currently saving or
+ * loading, when an IO data type callback is called.  There is no guarantee
+ * that the key name is always available, so this may return NULL.
+ */
+const RedisModuleString *RM_GetKeyNameFromIO(RedisModuleIO *io) {
+    return io->key;
+}
+
 /* --------------------------------------------------------------------------
  * Logging
  * -------------------------------------------------------------------------- */
@@ -5164,6 +5172,7 @@ void moduleRegisterCoreAPI(void) {
     REGISTER_API(RetainString);
     REGISTER_API(StringCompare);
     REGISTER_API(GetContextFromIO);
+    REGISTER_API(GetKeyNameFromIO);
     REGISTER_API(BlockClient);
     REGISTER_API(UnblockClient);
     REGISTER_API(IsBlockedReplyRequest);
diff --git a/src/rdb.c b/src/rdb.c
index 52dddf210..95e4766ea 100644
--- a/src/rdb.c
+++ b/src/rdb.c
@@ -751,7 +751,7 @@ size_t rdbSaveStreamConsumers(rio *rdb, streamCG *cg) {
 
 /* Save a Redis object.
  * Returns -1 on error, number of bytes written on success. */
-ssize_t rdbSaveObject(rio *rdb, robj *o) {
+ssize_t rdbSaveObject(rio *rdb, robj *o, robj *key) {
     ssize_t n = 0, nwritten = 0;
 
     if (o->type == OBJ_STRING) {
@@ -966,7 +966,7 @@ ssize_t rdbSaveObject(rio *rdb, robj *o) {
         RedisModuleIO io;
         moduleValue *mv = o->ptr;
         moduleType *mt = mv->type;
-        moduleInitIOContext(io,mt,rdb);
+        moduleInitIOContext(io,mt,rdb,key);
 
         /* Write the "module" identifier as prefix, so that we'll be able
          * to call the right module during loading. */
@@ -996,7 +996,7 @@ ssize_t rdbSaveObject(rio *rdb, robj *o) {
  * this length with very little changes to the code. In the future
  * we could switch to a faster solution. */
 size_t rdbSavedObjectLen(robj *o) {
-    ssize_t len = rdbSaveObject(NULL,o);
+    ssize_t len = rdbSaveObject(NULL,o,NULL);
     serverAssertWithInfo(NULL,o,len != -1);
     return len;
 }
@@ -1038,7 +1038,7 @@ int rdbSaveKeyValuePair(rio *rdb, robj *key, robj *val, long long expiretime) {
     /* Save type, key, value */
     if (rdbSaveObjectType(rdb,val) == -1) return -1;
     if (rdbSaveStringObject(rdb,key) == -1) return -1;
-    if (rdbSaveObject(rdb,val) == -1) return -1;
+    if (rdbSaveObject(rdb,val,key) == -1) return -1;
     return 1;
 }
 
@@ -1380,7 +1380,7 @@ robj *rdbLoadCheckModuleValue(rio *rdb, char *modulename) {
 
 /* Load a Redis object of the specified type from the specified file.
  * On success a newly allocated object is returned, otherwise NULL. */
-robj *rdbLoadObject(int rdbtype, rio *rdb) {
+robj *rdbLoadObject(int rdbtype, rio *rdb, robj *key) {
     robj *o = NULL, *ele, *dec;
     uint64_t len;
     unsigned int i;
@@ -1767,7 +1767,7 @@ robj *rdbLoadObject(int rdbtype, rio *rdb) {
             exit(1);
         }
         RedisModuleIO io;
-        moduleInitIOContext(io,mt,rdb);
+        moduleInitIOContext(io,mt,rdb,key);
         io.ver = (rdbtype == RDB_TYPE_MODULE) ? 1 : 2;
         /* Call the rdb_load method of the module providing the 10 bit
          * encoding version in the lower 10 bits of the module ID. */
@@ -2023,7 +2023,7 @@ int rdbLoadRio(rio *rdb, rdbSaveInfo *rsi, int loading_aof) {
         /* Read key */
         if ((key = rdbLoadStringObject(rdb)) == NULL) goto eoferr;
         /* Read value */
-        if ((val = rdbLoadObject(type,rdb)) == NULL) goto eoferr;
+        if ((val = rdbLoadObject(type,rdb,key)) == NULL) goto eoferr;
         /* Check if the key already expired. This function is used when loading
          * an RDB file from disk, either at startup, or when an RDB was
          * received from the master. In the latter case, the master is
diff --git a/src/rdb.h b/src/rdb.h
index 7b9486169..0acddf9ab 100644
--- a/src/rdb.h
+++ b/src/rdb.h
@@ -140,9 +140,9 @@ int rdbSaveBackground(char *filename, rdbSaveInfo *rsi);
 int rdbSaveToSlavesSockets(rdbSaveInfo *rsi);
 void rdbRemoveTempFile(pid_t childpid);
 int rdbSave(char *filename, rdbSaveInfo *rsi);
-ssize_t rdbSaveObject(rio *rdb, robj *o);
+ssize_t rdbSaveObject(rio *rdb, robj *o, robj *key);
 size_t rdbSavedObjectLen(robj *o);
-robj *rdbLoadObject(int type, rio *rdb);
+robj *rdbLoadObject(int type, rio *rdb, robj *key);
 void backgroundSaveDoneHandler(int exitcode, int bysignal);
 int rdbSaveKeyValuePair(rio *rdb, robj *key, robj *val, long long expiretime);
 robj *rdbLoadStringObject(rio *rdb);
diff --git a/src/redis-check-rdb.c b/src/redis-check-rdb.c
index 8de1d8f48..ec00ee71c 100644
--- a/src/redis-check-rdb.c
+++ b/src/redis-check-rdb.c
@@ -285,7 +285,7 @@ int redis_check_rdb(char *rdbfilename, FILE *fp) {
         rdbstate.keys++;
         /* Read value */
         rdbstate.doing = RDB_CHECK_DOING_READ_OBJECT_VALUE;
-        if ((val = rdbLoadObject(type,&rdb)) == NULL) goto eoferr;
+        if ((val = rdbLoadObject(type,&rdb,key)) == NULL) goto eoferr;
         /* Check if the key already expired. */
         if (expiretime != -1 && expiretime < now)
             rdbstate.already_expired++;
diff --git a/src/redismodule.h b/src/redismodule.h
index 272da08df..02941aa96 100644
--- a/src/redismodule.h
+++ b/src/redismodule.h
@@ -278,6 +278,7 @@ int REDISMODULE_API_FUNC(RedisModule_StringAppendBuffer)(RedisModuleCtx *ctx, Re
 void REDISMODULE_API_FUNC(RedisModule_RetainString)(RedisModuleCtx *ctx, RedisModuleString *str);
 int REDISMODULE_API_FUNC(RedisModule_StringCompare)(RedisModuleString *a, RedisModuleString *b);
 RedisModuleCtx *REDISMODULE_API_FUNC(RedisModule_GetContextFromIO)(RedisModuleIO *io);
+const RedisModuleString *REDISMODULE_API_FUNC(RedisModule_GetKeyNameFromIO)(RedisModuleIO *io);
 long long REDISMODULE_API_FUNC(RedisModule_Milliseconds)(void);
 void REDISMODULE_API_FUNC(RedisModule_DigestAddStringBuffer)(RedisModuleDigest *md, unsigned char *ele, size_t len);
 void REDISMODULE_API_FUNC(RedisModule_DigestAddLongLong)(RedisModuleDigest *md, long long ele);
@@ -442,6 +443,7 @@ static int RedisModule_Init(RedisModuleCtx *ctx, const char *name, int ver, int
     REDISMODULE_GET_API(RetainString);
     REDISMODULE_GET_API(StringCompare);
     REDISMODULE_GET_API(GetContextFromIO);
+    REDISMODULE_GET_API(GetKeyNameFromIO);
     REDISMODULE_GET_API(Milliseconds);
     REDISMODULE_GET_API(DigestAddStringBuffer);
     REDISMODULE_GET_API(DigestAddLongLong);
diff --git a/src/server.h b/src/server.h
index 56c3b67d3..b888266a4 100644
--- a/src/server.h
+++ b/src/server.h
@@ -578,16 +578,18 @@ typedef struct RedisModuleIO {
     int ver;            /* Module serialization version: 1 (old),
                          * 2 (current version with opcodes annotation). */
     struct RedisModuleCtx *ctx; /* Optional context, see RM_GetContextFromIO()*/
+    struct redisObject *key;    /* Optional name of key processed */
 } RedisModuleIO;
 
 /* Macro to initialize an IO context. Note that the 'ver' field is populated
  * inside rdb.c according to the version of the value to load. */
-#define moduleInitIOContext(iovar,mtype,rioptr) do { \
+#define moduleInitIOContext(iovar,mtype,rioptr,keyptr) do { \
     iovar.rio = rioptr; \
     iovar.type = mtype; \
     iovar.bytes = 0; \
     iovar.error = 0; \
     iovar.ver = 0; \
+    iovar.key = keyptr; \
     iovar.ctx = NULL; \
 } while(0);
 

From 7f6aaa6f650b5ab51f02111e07b3c68c06b9856b Mon Sep 17 00:00:00 2001
From: John Sully <john@csquare.ca>
Date: Thu, 14 Mar 2019 14:02:16 -0400
Subject: [PATCH 21/60] Fix hyperloglog corruption

---
 src/hyperloglog.c | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/src/hyperloglog.c b/src/hyperloglog.c
index fc21ea006..e993bf26e 100644
--- a/src/hyperloglog.c
+++ b/src/hyperloglog.c
@@ -614,6 +614,10 @@ int hllSparseToDense(robj *o) {
         } else {
             runlen = HLL_SPARSE_VAL_LEN(p);
             regval = HLL_SPARSE_VAL_VALUE(p);
+            if ((runlen + idx) > HLL_REGISTERS) {
+                sdsfree(dense);
+                return C_ERR;
+            }
             while(runlen--) {
                 HLL_DENSE_SET_REGISTER(hdr->registers,idx,regval);
                 idx++;
@@ -1088,6 +1092,8 @@ int hllMerge(uint8_t *max, robj *hll) {
             } else {
                 runlen = HLL_SPARSE_VAL_LEN(p);
                 regval = HLL_SPARSE_VAL_VALUE(p);
+                if ((runlen + i) > HLL_REGISTERS)
+                    return C_ERR;
                 while(runlen--) {
                     if (regval > max[i]) max[i] = regval;
                     i++;

From a47cd50a0d7478d495612f4c4d166ba564381f04 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Fri, 15 Mar 2019 13:52:29 +0100
Subject: [PATCH 22/60] HyperLogLog: dense/sparse repr parsing fuzz test.

---
 tests/unit/hyperloglog.tcl | 29 +++++++++++++++++++++++++++++
 1 file changed, 29 insertions(+)

diff --git a/tests/unit/hyperloglog.tcl b/tests/unit/hyperloglog.tcl
index 7d36b7a35..6a9c47b11 100644
--- a/tests/unit/hyperloglog.tcl
+++ b/tests/unit/hyperloglog.tcl
@@ -115,6 +115,35 @@ start_server {tags {"hll"}} {
         set e
     } {*WRONGTYPE*}
 
+    test {Fuzzing dense/sparse encoding: Redis should always detect errors} {
+        for {set j 0} {$j < 10000} {incr j} {
+            r del hll
+            set items {}
+            set numitems [randomInt 3000]
+            for {set i 0} {$i < $numitems} {incr i} {
+                lappend items [expr {rand()}]
+            }
+            r pfadd hll {*}$items
+
+            # Corrupt it in some random way.
+            for {set i 0} {$i < 5} {incr i} {
+                set len [r strlen hll]
+                set pos [randomInt $len]
+                set byte [randstring 1 1 binary]
+                r setrange hll $pos $byte
+                # Don't modify more bytes 50% of times
+                if {rand() < 0.5} break
+            }
+
+            # Use the hyperloglog to check if it crashes
+            # Redis in some way.
+            catch {
+                r pfcount hll
+                r pfdebug getreg hll
+            }
+        }
+    }
+
     test {PFADD, PFCOUNT, PFMERGE type checking works} {
         r set foo bar
         catch {r pfadd foo 1} e

From 3765dfdcf76fa37143457580b31c176bc3100dc3 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Fri, 15 Mar 2019 17:10:16 +0100
Subject: [PATCH 23/60] HyperLogLog: enlarge reghisto variable for safety.

---
 src/hyperloglog.c | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/src/hyperloglog.c b/src/hyperloglog.c
index e993bf26e..526510b43 100644
--- a/src/hyperloglog.c
+++ b/src/hyperloglog.c
@@ -1017,7 +1017,12 @@ uint64_t hllCount(struct hllhdr *hdr, int *invalid) {
     double m = HLL_REGISTERS;
     double E;
     int j;
-    int reghisto[HLL_Q+2] = {0};
+    /* Note that reghisto could be just HLL_Q+1, becuase this is the
+     * maximum frequency of the "000...1" sequence the hash function is
+     * able to return. However it is slow to check for sanity of the
+     * input: instead we history array at a safe size: overflows will
+     * just write data to wrong, but correctly allocated, places. */
+    int reghisto[64] = {0};
 
     /* Compute register histogram */
     if (hdr->encoding == HLL_DENSE) {

From ca291b0951ff4dd4db4c08e3fe7bea7311d94e47 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Fri, 15 Mar 2019 17:13:19 +0100
Subject: [PATCH 24/60] HyperLogLog: speedup fuzz test.

---
 tests/unit/hyperloglog.tcl | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/tests/unit/hyperloglog.tcl b/tests/unit/hyperloglog.tcl
index 6a9c47b11..712fcc641 100644
--- a/tests/unit/hyperloglog.tcl
+++ b/tests/unit/hyperloglog.tcl
@@ -116,7 +116,7 @@ start_server {tags {"hll"}} {
     } {*WRONGTYPE*}
 
     test {Fuzzing dense/sparse encoding: Redis should always detect errors} {
-        for {set j 0} {$j < 10000} {incr j} {
+        for {set j 0} {$j < 1000} {incr j} {
             r del hll
             set items {}
             set numitems [randomInt 3000]
@@ -139,7 +139,6 @@ start_server {tags {"hll"}} {
             # Redis in some way.
             catch {
                 r pfcount hll
-                r pfdebug getreg hll
             }
         }
     }

From 961aa74ff1710b48d0f55da175e0f7665ccb197e Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Fri, 15 Mar 2019 17:16:06 +0100
Subject: [PATCH 25/60] HyperLogLog: handle wrong offset in the base case.

---
 src/hyperloglog.c | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/src/hyperloglog.c b/src/hyperloglog.c
index 526510b43..1e7ce3dce 100644
--- a/src/hyperloglog.c
+++ b/src/hyperloglog.c
@@ -614,10 +614,7 @@ int hllSparseToDense(robj *o) {
         } else {
             runlen = HLL_SPARSE_VAL_LEN(p);
             regval = HLL_SPARSE_VAL_VALUE(p);
-            if ((runlen + idx) > HLL_REGISTERS) {
-                sdsfree(dense);
-                return C_ERR;
-            }
+            if ((runlen + idx) > HLL_REGISTERS) break; /* Overflow. */
             while(runlen--) {
                 HLL_DENSE_SET_REGISTER(hdr->registers,idx,regval);
                 idx++;
@@ -1097,8 +1094,7 @@ int hllMerge(uint8_t *max, robj *hll) {
             } else {
                 runlen = HLL_SPARSE_VAL_LEN(p);
                 regval = HLL_SPARSE_VAL_VALUE(p);
-                if ((runlen + i) > HLL_REGISTERS)
-                    return C_ERR;
+                if ((runlen + i) > HLL_REGISTERS) break; /* Overflow. */
                 while(runlen--) {
                     if (regval > max[i]) max[i] = regval;
                     i++;

From 218091d2e4eb8704d2caccc8f30d12dee9435e2f Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Sat, 16 Mar 2019 09:15:12 +0100
Subject: [PATCH 26/60] HyperLogLog: fix comment in hllCount().

---
 src/hyperloglog.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/hyperloglog.c b/src/hyperloglog.c
index 1e7ce3dce..e01ea6042 100644
--- a/src/hyperloglog.c
+++ b/src/hyperloglog.c
@@ -1014,8 +1014,8 @@ uint64_t hllCount(struct hllhdr *hdr, int *invalid) {
     double m = HLL_REGISTERS;
     double E;
     int j;
-    /* Note that reghisto could be just HLL_Q+1, becuase this is the
-     * maximum frequency of the "000...1" sequence the hash function is
+    /* Note that reghisto size could be just HLL_Q+2, becuase HLL_Q+1 is
+     * the maximum frequency of the "000...1" sequence the hash function is
      * able to return. However it is slow to check for sanity of the
      * input: instead we history array at a safe size: overflows will
      * just write data to wrong, but correctly allocated, places. */

From 71d9f4e59606270fb8862803ded3f0bfdd7bb03b Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Mon, 18 Mar 2019 11:15:39 +0100
Subject: [PATCH 27/60] redis-check-aof: fix potential overflow.

Bug signaled by @vattezhang in PR #5940 but fixed differently.
---
 src/redis-check-aof.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/redis-check-aof.c b/src/redis-check-aof.c
index c4d5a225e..54ed85f0d 100644
--- a/src/redis-check-aof.c
+++ b/src/redis-check-aof.c
@@ -33,8 +33,8 @@
 
 #define ERROR(...) { \
     char __buf[1024]; \
-    sprintf(__buf, __VA_ARGS__); \
-    sprintf(error, "0x%16llx: %s", (long long)epos, __buf); \
+    snprintf(__buf, sizeof(__buf), __VA_ARGS__); \
+    snprintf(error, sizeof(error), "0x%16llx: %s", (long long)epos, __buf); \
 }
 
 static char error[1024];

From 340d03b64bf59a6c0a79a86a51e823397d5a9b57 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Mon, 18 Mar 2019 11:34:40 +0100
Subject: [PATCH 28/60] replicaofCommand() refactoring: stay into 80 cols.

---
 src/replication.c | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/src/replication.c b/src/replication.c
index 3c30999af..f2adc7995 100644
--- a/src/replication.c
+++ b/src/replication.c
@@ -2053,8 +2053,11 @@ void replicaofCommand(client *c) {
         /* Check if we are already attached to the specified slave */
         if (server.masterhost && !strcasecmp(server.masterhost,c->argv[1]->ptr)
             && server.masterport == port) {
-            serverLog(LL_NOTICE,"REPLICAOF would result into synchronization with the master we are already connected with. No operation performed.");
-            addReplySds(c,sdsnew("+OK Already connected to specified master\r\n"));
+            serverLog(LL_NOTICE,"REPLICAOF would result into synchronization "
+                                "with the master we are already connected "
+                                "with. No operation performed.");
+            addReplySds(c,sdsnew("+OK Already connected to specified "
+                                 "master\r\n"));
             return;
         }
         /* There was no previous master or the user specified a different one,

From bb6e8ba6829eab24444d6fe6950160d05be7d96c Mon Sep 17 00:00:00 2001
From: Yossi Gottlieb <yossigo@gmail.com>
Date: Fri, 23 Feb 2018 16:19:37 +0200
Subject: [PATCH 29/60] Initial command filter experiment.

---
 src/module.c              | 76 +++++++++++++++++++++++++++++++++++++++
 src/modules/Makefile      |  6 +++-
 src/modules/hellofilter.c | 69 +++++++++++++++++++++++++++++++++++
 src/redismodule.h         |  8 +++++
 src/server.c              |  2 ++
 src/server.h              |  2 +-
 6 files changed, 161 insertions(+), 2 deletions(-)
 create mode 100644 src/modules/hellofilter.c

diff --git a/src/module.c b/src/module.c
index e69d3dc61..1780342ed 100644
--- a/src/module.c
+++ b/src/module.c
@@ -270,6 +270,28 @@ typedef struct RedisModuleDictIter {
     raxIterator ri;
 } RedisModuleDictIter;
 
+/* Information about the command to be executed, as passed to and from a
+ * filter. */
+typedef struct RedisModuleFilteredCommand {
+    RedisModuleString **argv;
+    int argc;
+} RedisModuleFilteredCommand;
+
+typedef void (*RedisModuleCommandFilterFunc) (RedisModuleCtx *ctx, RedisModuleFilteredCommand *cmd);
+
+typedef struct RedisModuleCommandFilter {
+    /* The module that registered the filter */
+    RedisModule *module;
+    /* Filter callback function */
+    RedisModuleCommandFilterFunc callback;
+    /* Indicates a filter is active, avoid reentrancy */
+    int active;
+} RedisModuleCommandFilter;
+
+/* Registered filters */
+static list *moduleCommandFilters;
+
+
 /* --------------------------------------------------------------------------
  * Prototypes
  * -------------------------------------------------------------------------- */
@@ -4770,6 +4792,56 @@ int moduleUnregisterUsedAPI(RedisModule *module) {
     return count;
 }
 
+/* --------------------------------------------------------------------------
+ * Module Command Filter API
+ * -------------------------------------------------------------------------- */
+
+/* Register a new command filter function.  Filters get executed by Redis
+ * before processing an inbound command and can be used to manipulate the
+ * behavior of standard Redis commands.  Filters must not attempt to
+ * perform Redis commands or operate on the dataset, and must restrict
+ * themselves to manipulation of the arguments.
+ */
+
+int RM_RegisterCommandFilter(RedisModuleCtx *ctx, RedisModuleCommandFilterFunc callback) {
+    RedisModuleCommandFilter *filter = zmalloc(sizeof(*filter));
+    filter->module = ctx->module;
+    filter->callback = callback;
+    filter->active = 0;
+
+    listAddNodeTail(moduleCommandFilters, filter);
+    return REDISMODULE_OK;
+}
+
+void moduleCallCommandFilters(client *c) {
+    if (listLength(moduleCommandFilters) == 0) return;
+
+    listIter li;
+    listNode *ln;
+    listRewind(moduleCommandFilters,&li);
+
+    RedisModuleFilteredCommand cmd = {
+        .argv = c->argv,
+        .argc = c->argc
+    };
+
+    while((ln = listNext(&li))) {
+        RedisModuleCommandFilter *filter = ln->value;
+        if (filter->active) continue;
+
+        RedisModuleCtx ctx = REDISMODULE_CTX_INIT;
+        ctx.module = filter->module;
+
+        filter->active = 1;
+        filter->callback(&ctx, &cmd);
+        filter->active = 0;
+        moduleFreeContext(&ctx);
+    }
+
+    c->argv = cmd.argv;
+    c->argc = cmd.argc;
+}
+
 /* --------------------------------------------------------------------------
  * Modules API internals
  * -------------------------------------------------------------------------- */
@@ -4816,6 +4888,9 @@ void moduleInitModulesSystem(void) {
     moduleFreeContextReusedClient->flags |= CLIENT_MODULE;
     moduleFreeContextReusedClient->user = NULL; /* root user. */
 
+    /* Set up filter list */
+    moduleCommandFilters = listCreate();
+
     moduleRegisterCoreAPI();
     if (pipe(server.module_blocked_pipe) == -1) {
         serverLog(LL_WARNING,
@@ -5219,4 +5294,5 @@ void moduleRegisterCoreAPI(void) {
     REGISTER_API(DictCompare);
     REGISTER_API(ExportSharedAPI);
     REGISTER_API(GetSharedAPI);
+    REGISTER_API(RegisterCommandFilter);
 }
diff --git a/src/modules/Makefile b/src/modules/Makefile
index 51ffac17d..537aa0daf 100644
--- a/src/modules/Makefile
+++ b/src/modules/Makefile
@@ -13,7 +13,7 @@ endif
 
 .SUFFIXES: .c .so .xo .o
 
-all: helloworld.so hellotype.so helloblock.so testmodule.so hellocluster.so hellotimer.so hellodict.so
+all: helloworld.so hellotype.so helloblock.so testmodule.so hellocluster.so hellotimer.so hellodict.so hellofilter.so
 
 .c.xo:
 	$(CC) -I. $(CFLAGS) $(SHOBJ_CFLAGS) -fPIC -c $< -o $@
@@ -46,6 +46,10 @@ hellotimer.so: hellotimer.xo
 hellodict.xo: ../redismodule.h
 
 hellodict.so: hellodict.xo
+
+hellofilter.xo: ../redismodule.h
+
+hellofilter.so: hellofilter.xo
 	$(LD) -o $@ $< $(SHOBJ_LDFLAGS) $(LIBS) -lc
 
 testmodule.xo: ../redismodule.h
diff --git a/src/modules/hellofilter.c b/src/modules/hellofilter.c
new file mode 100644
index 000000000..c9e33158f
--- /dev/null
+++ b/src/modules/hellofilter.c
@@ -0,0 +1,69 @@
+#define REDISMODULE_EXPERIMENTAL_API
+#include "../redismodule.h"
+
+static RedisModuleString *log_key_name;
+
+static const char log_command_name[] = "hellofilter.log";
+
+int HelloFilter_LogCommand(RedisModuleCtx *ctx, RedisModuleString **argv, int argc)
+{
+    RedisModuleString *s = RedisModule_CreateStringFromString(ctx, argv[0]);
+
+    int i;
+    for (i = 1; i < argc; i++) {
+        size_t arglen;
+        const char *arg = RedisModule_StringPtrLen(argv[i], &arglen);
+
+        RedisModule_StringAppendBuffer(ctx, s, " ", 1);
+        RedisModule_StringAppendBuffer(ctx, s, arg, arglen);
+    }
+
+    RedisModuleKey *log = RedisModule_OpenKey(ctx, log_key_name, REDISMODULE_WRITE|REDISMODULE_READ);
+    RedisModule_ListPush(log, REDISMODULE_LIST_HEAD, s);
+    RedisModule_CloseKey(log);
+    RedisModule_FreeString(ctx, s);
+
+    size_t cmdlen;
+    const char *cmdname = RedisModule_StringPtrLen(argv[1], &cmdlen);
+    RedisModuleCallReply *reply = RedisModule_Call(ctx, cmdname, "v", &argv[2], argc - 2);
+    if (reply) {
+        RedisModule_ReplyWithCallReply(ctx, reply);
+        RedisModule_FreeCallReply(reply);
+    } else {
+        RedisModule_ReplyWithSimpleString(ctx, "Unknown command or invalid arguments");
+    }
+    return REDISMODULE_OK;
+}
+
+void HelloFilter_CommandFilter(RedisModuleCtx *ctx, RedisModuleFilteredCommand *cmd)
+{
+    cmd->argv = RedisModule_Realloc(cmd->argv, (cmd->argc+1)*sizeof(RedisModuleString *));
+    int i;
+
+    for (i = cmd->argc; i > 0; i--) {
+        cmd->argv[i] = cmd->argv[i-1];
+    }
+    cmd->argv[0] = RedisModule_CreateString(ctx, log_command_name, sizeof(log_command_name)-1);
+    cmd->argc++;
+}
+
+int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
+    if (RedisModule_Init(ctx,"hellofilter",1,REDISMODULE_APIVER_1)
+            == REDISMODULE_ERR) return REDISMODULE_ERR;
+
+    if (argc != 1) {
+        RedisModule_Log(ctx, "warning", "Log key name not specified");
+        return REDISMODULE_ERR;
+    }
+
+    log_key_name = RedisModule_CreateStringFromString(ctx, argv[0]);
+
+    if (RedisModule_CreateCommand(ctx,log_command_name,
+                HelloFilter_LogCommand,"write deny-oom",1,1,1) == REDISMODULE_ERR)
+            return REDISMODULE_ERR;
+
+    if (RedisModule_RegisterCommandFilter(ctx, HelloFilter_CommandFilter)
+            == REDISMODULE_ERR) return REDISMODULE_ERR;
+
+    return REDISMODULE_OK;
+}
diff --git a/src/redismodule.h b/src/redismodule.h
index 272da08df..54ce99d96 100644
--- a/src/redismodule.h
+++ b/src/redismodule.h
@@ -163,6 +163,12 @@ typedef void (*RedisModuleTypeFreeFunc)(void *value);
 typedef void (*RedisModuleClusterMessageReceiver)(RedisModuleCtx *ctx, const char *sender_id, uint8_t type, const unsigned char *payload, uint32_t len);
 typedef void (*RedisModuleTimerProc)(RedisModuleCtx *ctx, void *data);
 
+typedef struct RedisModuleFilteredCommand {
+    RedisModuleString **argv;
+    int argc;
+} RedisModuleFilteredCommand;
+typedef void (*RedisModuleCommandFilterFunc) (RedisModuleCtx *ctx, RedisModuleFilteredCommand *cmd);
+
 #define REDISMODULE_TYPE_METHOD_VERSION 1
 typedef struct RedisModuleTypeMethods {
     uint64_t version;
@@ -337,6 +343,7 @@ void REDISMODULE_API_FUNC(RedisModule_SetDisconnectCallback)(RedisModuleBlockedC
 void REDISMODULE_API_FUNC(RedisModule_SetClusterFlags)(RedisModuleCtx *ctx, uint64_t flags);
 int REDISMODULE_API_FUNC(RedisModule_ExportSharedAPI)(RedisModuleCtx *ctx, const char *apiname, void *func);
 void *REDISMODULE_API_FUNC(RedisModule_GetSharedAPI)(RedisModuleCtx *ctx, const char *apiname);
+int REDISMODULE_API_FUNC(RedisModule_RegisterCommandFilter)(RedisModuleCtx *ctx, RedisModuleCommandFilterFunc cb);
 #endif
 
 /* This is included inline inside each Redis module. */
@@ -499,6 +506,7 @@ static int RedisModule_Init(RedisModuleCtx *ctx, const char *name, int ver, int
     REDISMODULE_GET_API(SetClusterFlags);
     REDISMODULE_GET_API(ExportSharedAPI);
     REDISMODULE_GET_API(GetSharedAPI);
+    REDISMODULE_GET_API(RegisterCommandFilter);
 #endif
 
     if (RedisModule_IsModuleNameBusy && RedisModule_IsModuleNameBusy(name)) return REDISMODULE_ERR;
diff --git a/src/server.c b/src/server.c
index 712cda1bd..66e79dea3 100644
--- a/src/server.c
+++ b/src/server.c
@@ -3268,6 +3268,8 @@ void call(client *c, int flags) {
  * other operations can be performed by the caller. Otherwise
  * if C_ERR is returned the client was destroyed (i.e. after QUIT). */
 int processCommand(client *c) {
+    moduleCallCommandFilters(c);
+
     /* The QUIT command is handled separately. Normal command procs will
      * go through checking for replication and QUIT will cause trouble
      * when FORCE_REPLICATION is enabled and would be implemented in
diff --git a/src/server.h b/src/server.h
index 56c3b67d3..f55213bfc 100644
--- a/src/server.h
+++ b/src/server.h
@@ -1489,7 +1489,7 @@ size_t moduleCount(void);
 void moduleAcquireGIL(void);
 void moduleReleaseGIL(void);
 void moduleNotifyKeyspaceEvent(int type, const char *event, robj *key, int dbid);
-
+void moduleCallCommandFilters(client *c);
 
 /* Utils */
 long long ustime(void);

From a3b442eb5daece99f5ac52a167ab455d302eda43 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Mon, 18 Mar 2019 15:38:43 +0100
Subject: [PATCH 30/60] MANIFESTO v2.

---
 MANIFESTO | 47 +++++++++++++++++++++++++++++++++++++++++------
 1 file changed, 41 insertions(+), 6 deletions(-)

diff --git a/MANIFESTO b/MANIFESTO
index 2b719057e..d43a58893 100644
--- a/MANIFESTO
+++ b/MANIFESTO
@@ -34,7 +34,21 @@ Redis Manifesto
     so that the complexity is obvious and more complex operations can be
     performed as the sum of the basic operations.
 
-4 - Code is like a poem; it's not just something we write to reach some
+4 - We believe in code efficiency. Computers get faster and faster, yet we
+    believe that abusing computing capabilities is not wise: the amount of
+    operations you can do for a given amount of energy remains anyway a
+    significant parameter: it allows to do more with less computers and, at
+    the same time, having a smaller environmental impact. Similarly Redis is
+    able to "scale down" to smaller devices. It is perfectly usable in a
+    Raspberry Pi and other small ARM based computers. Faster code having
+    just the layers of abstractions that are really needed will also result,
+    often, in more predictable performances. We think likewise about memory
+    usage, one of the fundamental goals of the Redis project is to
+    incrementally build more and more memory efficient data structures, so that
+    problems that were not approachable in RAM in the past will be perfectly
+    fine to handle in the future.
+
+5 - Code is like a poem; it's not just something we write to reach some
     practical result. Sometimes people that are far from the Redis philosophy
     suggest using other code written by other authors (frequently in other
     languages) in order to implement something Redis currently lacks. But to us
@@ -45,23 +59,44 @@ Redis Manifesto
     when needed. At the same time, when writing the Redis story we're trying to
     write smaller stories that will fit in to other code.
 
-5 - We're against complexity. We believe designing systems is a fight against
+6 - We're against complexity. We believe designing systems is a fight against
     complexity. We'll accept to fight the complexity when it's worthwhile but
     we'll try hard to recognize when a small feature is not worth 1000s of lines
     of code. Most of the time the best way to fight complexity is by not
     creating it at all.
 
-6 - Two levels of API. The Redis API has two levels: 1) a subset of the API fits
+7 - Threading is not a silver bullet. Instead of making Redis threaded we
+    believe on the idea of an efficient (mostly) single threaded Redis core.
+    Multiple of such cores, that may run in the same computer or may run
+    in multiple computers, are abstracted away as a single big system by
+    higher order protocols and features: Redis Cluster and the upcoming
+    Redis Proxy are our main goals. A shared nothing approach is not just
+    much simpler (see the previous point in this document), is also optimal
+    in NUMA systems. In the specific case of Redis it allows for each instance
+    to have a more limited amount of data, making the Redis persist-by-fork
+    approach more sounding. In the future we may explore parallelism only for
+    I/O, which is the low hanging fruit: minimal complexity could provide an
+    improved single process experience.
+
+8 - Two levels of API. The Redis API has two levels: 1) a subset of the API fits
     naturally into a distributed version of Redis and 2) a more complex API that
     supports multi-key operations. Both are useful if used judiciously but
     there's no way to make the more complex multi-keys API distributed in an
     opaque way without violating our other principles. We don't want to provide
     the illusion of something that will work magically when actually it can't in
     all cases. Instead we'll provide commands to quickly migrate keys from one
-    instance to another to perform multi-key operations and expose the tradeoffs
-    to the user.
+    instance to another to perform multi-key operations and expose the
+    trade-offs to the user.
 
-7 - We optimize for joy. We believe writing code is a lot of hard work, and the
+9 - We optimize for joy. We believe writing code is a lot of hard work, and the
     only way it can be worth is by enjoying it. When there is no longer joy in
     writing code, the best thing to do is stop. To prevent this, we'll avoid
     taking paths that will make Redis less of a joy to develop.
+
+10 - All the above points are put together in what we call opportunistic
+     programming: trying to get the most for the user with minimal increases
+     in complexity (hanging fruits). Solve 95% of the problem with 5% of the
+     code when it is acceptable. Avoid a fixed schedule but follow the flow of
+     user requests, inspiration, Redis internal readiness for certain features
+     (sometimes many past changes reach a critical point making a previously
+     complex feature very easy to obtain).

From a40a075ada7246b858bda75c8dca4a05f0bcf8b8 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Mon, 18 Mar 2019 15:49:52 +0100
Subject: [PATCH 31/60] MANIFESTO: simplicity and lock-in.

---
 MANIFESTO | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/MANIFESTO b/MANIFESTO
index d43a58893..372789462 100644
--- a/MANIFESTO
+++ b/MANIFESTO
@@ -63,7 +63,11 @@ Redis Manifesto
     complexity. We'll accept to fight the complexity when it's worthwhile but
     we'll try hard to recognize when a small feature is not worth 1000s of lines
     of code. Most of the time the best way to fight complexity is by not
-    creating it at all.
+    creating it at all. Complexity is also a form of lock-in: code that is
+    very hard to understand cannot be modified by users in an independent way
+    regardless of the license. One of the main Redis goals is to remain
+    understandable, enough for a single programmer to have a clear idea of how
+    it works in detail just reading the source code for a couple of weeks.
 
 7 - Threading is not a silver bullet. Instead of making Redis threaded we
     believe on the idea of an efficient (mostly) single threaded Redis core.

From bc47c987d68ade4f481122cfd6cec639dbba91ad Mon Sep 17 00:00:00 2001
From: Yossi Gottlieb <yossigo@gmail.com>
Date: Mon, 18 Mar 2019 18:36:46 +0200
Subject: [PATCH 32/60] Add command filtering argument handling API.

---
 src/module.c              | 81 +++++++++++++++++++++++++++++++++++++++
 src/modules/hellofilter.c | 46 ++++++++++++++++++----
 src/redismodule.h         | 18 ++++++---
 3 files changed, 132 insertions(+), 13 deletions(-)

diff --git a/src/module.c b/src/module.c
index 1780342ed..741c546b7 100644
--- a/src/module.c
+++ b/src/module.c
@@ -291,6 +291,10 @@ typedef struct RedisModuleCommandFilter {
 /* Registered filters */
 static list *moduleCommandFilters;
 
+typedef struct RedisModuleCommandFilterCtx {
+    RedisModuleString **argv;
+    int argc;
+} RedisModuleCommandFilterCtx;
 
 /* --------------------------------------------------------------------------
  * Prototypes
@@ -4842,6 +4846,78 @@ void moduleCallCommandFilters(client *c) {
     c->argc = cmd.argc;
 }
 
+/* Return the number of arguments a filtered command has.  The number of
+ * arguments include the command itself.
+ */
+int RM_CommandFilterArgsCount(RedisModuleCommandFilterCtx *filter)
+{
+    return filter->argc;
+}
+
+/* Return the specified command argument.  The first argument (position 0) is
+ * the command itself, and the rest are user-provided args.
+ */
+const RedisModuleString *RM_CommandFilterArgGet(RedisModuleCommandFilterCtx *filter, int pos)
+{
+    if (pos < 0 || pos >= filter->argc) return NULL;
+    return filter->argv[pos];
+}
+
+/* Modify the filtered command by inserting a new argument at the specified
+ * position.  The specified RedisModuleString argument may be used by Redis
+ * after the filter context is destroyed, so it must not be auto-memory
+ * allocated, freed or used elsewhere.
+ */
+
+int RM_CommandFilterArgInsert(RedisModuleCommandFilterCtx *filter, int pos, RedisModuleString *arg)
+{
+    int i;
+
+    if (pos < 0 || pos > filter->argc) return REDISMODULE_ERR;
+
+    filter->argv = zrealloc(filter->argv, (filter->argc+1)*sizeof(RedisModuleString *));
+    for (i = filter->argc; i > pos; i--) {
+        filter->argv[i] = filter->argv[i-1];
+    }
+    filter->argv[pos] = arg;
+    filter->argc++;
+
+    return REDISMODULE_OK;
+}
+
+/* Modify the filtered command by replacing an existing argument with a new one.
+ * The specified RedisModuleString argument may be used by Redis after the
+ * filter context is destroyed, so it must not be auto-memory allocated, freed
+ * or used elsewhere.
+ */
+
+int RM_CommandFilterArgReplace(RedisModuleCommandFilterCtx *filter, int pos, RedisModuleString *arg)
+{
+    if (pos < 0 || pos >= filter->argc) return REDISMODULE_ERR;
+    
+    decrRefCount(filter->argv[pos]);
+    filter->argv[pos] = arg;
+
+    return REDISMODULE_OK;
+}
+
+/* Modify the filtered command by deleting an argument at the specified
+ * position.
+ */
+int RM_CommandFilterArgDelete(RedisModuleCommandFilterCtx *filter, int pos)
+{
+    int i;
+    if (pos < 0 || pos >= filter->argc) return REDISMODULE_ERR;
+
+    decrRefCount(filter->argv[pos]);
+    for (i = pos; i < filter->argc-1; i++) {
+        filter->argv[i] = filter->argv[i+1];
+    }
+    filter->argc--;
+    
+    return REDISMODULE_OK;
+}
+
 /* --------------------------------------------------------------------------
  * Modules API internals
  * -------------------------------------------------------------------------- */
@@ -5295,4 +5371,9 @@ void moduleRegisterCoreAPI(void) {
     REGISTER_API(ExportSharedAPI);
     REGISTER_API(GetSharedAPI);
     REGISTER_API(RegisterCommandFilter);
+    REGISTER_API(CommandFilterArgsCount);
+    REGISTER_API(CommandFilterArgGet);
+    REGISTER_API(CommandFilterArgInsert);
+    REGISTER_API(CommandFilterArgReplace);
+    REGISTER_API(CommandFilterArgDelete);
 }
diff --git a/src/modules/hellofilter.c b/src/modules/hellofilter.c
index c9e33158f..84eb02c30 100644
--- a/src/modules/hellofilter.c
+++ b/src/modules/hellofilter.c
@@ -1,6 +1,8 @@
 #define REDISMODULE_EXPERIMENTAL_API
 #include "../redismodule.h"
 
+#include <string.h>
+
 static RedisModuleString *log_key_name;
 
 static const char log_command_name[] = "hellofilter.log";
@@ -35,16 +37,46 @@ int HelloFilter_LogCommand(RedisModuleCtx *ctx, RedisModuleString **argv, int ar
     return REDISMODULE_OK;
 }
 
-void HelloFilter_CommandFilter(RedisModuleCtx *ctx, RedisModuleFilteredCommand *cmd)
+void HelloFilter_CommandFilter(RedisModuleCtx *ctx, RedisModuleCommandFilterCtx *filter)
 {
-    cmd->argv = RedisModule_Realloc(cmd->argv, (cmd->argc+1)*sizeof(RedisModuleString *));
-    int i;
+    (void) ctx;
 
-    for (i = cmd->argc; i > 0; i--) {
-        cmd->argv[i] = cmd->argv[i-1];
+    /* Fun manipulations:
+     * - Remove @delme
+     * - Replace @replaceme
+     * - Append @insertbefore or @insertafter
+     * - Prefix with Log command if @log encounterd
+     */
+    int log = 0;
+    int pos = 0;
+    while (pos < RedisModule_CommandFilterArgsCount(filter)) {
+        const RedisModuleString *arg = RedisModule_CommandFilterArgGet(filter, pos);
+        size_t arg_len;
+        const char *arg_str = RedisModule_StringPtrLen(arg, &arg_len);
+
+        if (arg_len == 6 && !memcmp(arg_str, "@delme", 6)) {
+            RedisModule_CommandFilterArgDelete(filter, pos);
+            continue;
+        } 
+        if (arg_len == 10 && !memcmp(arg_str, "@replaceme", 10)) {
+            RedisModule_CommandFilterArgReplace(filter, pos,
+                    RedisModule_CreateString(NULL, "--replaced--", 12));
+        } else if (arg_len == 13 && !memcmp(arg_str, "@insertbefore", 13)) {
+            RedisModule_CommandFilterArgInsert(filter, pos,
+                    RedisModule_CreateString(NULL, "--inserted-before--", 19));
+            pos++;
+        } else if (arg_len == 12 && !memcmp(arg_str, "@insertafter", 12)) {
+            RedisModule_CommandFilterArgInsert(filter, pos + 1,
+                    RedisModule_CreateString(NULL, "--inserted-after--", 18));
+            pos++;
+        } else if (arg_len == 4 && !memcmp(arg_str, "@log", 4)) {
+            log = 1;
+        }
+        pos++;
     }
-    cmd->argv[0] = RedisModule_CreateString(ctx, log_command_name, sizeof(log_command_name)-1);
-    cmd->argc++;
+
+    if (log) RedisModule_CommandFilterArgInsert(filter, 0,
+            RedisModule_CreateString(NULL, log_command_name, sizeof(log_command_name)-1));
 }
 
 int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
diff --git a/src/redismodule.h b/src/redismodule.h
index 54ce99d96..426a6df69 100644
--- a/src/redismodule.h
+++ b/src/redismodule.h
@@ -150,6 +150,7 @@ typedef struct RedisModuleBlockedClient RedisModuleBlockedClient;
 typedef struct RedisModuleClusterInfo RedisModuleClusterInfo;
 typedef struct RedisModuleDict RedisModuleDict;
 typedef struct RedisModuleDictIter RedisModuleDictIter;
+typedef struct RedisModuleCommandFilterCtx RedisModuleCommandFilterCtx;
 
 typedef int (*RedisModuleCmdFunc)(RedisModuleCtx *ctx, RedisModuleString **argv, int argc);
 typedef void (*RedisModuleDisconnectFunc)(RedisModuleCtx *ctx, RedisModuleBlockedClient *bc);
@@ -162,12 +163,7 @@ typedef void (*RedisModuleTypeDigestFunc)(RedisModuleDigest *digest, void *value
 typedef void (*RedisModuleTypeFreeFunc)(void *value);
 typedef void (*RedisModuleClusterMessageReceiver)(RedisModuleCtx *ctx, const char *sender_id, uint8_t type, const unsigned char *payload, uint32_t len);
 typedef void (*RedisModuleTimerProc)(RedisModuleCtx *ctx, void *data);
-
-typedef struct RedisModuleFilteredCommand {
-    RedisModuleString **argv;
-    int argc;
-} RedisModuleFilteredCommand;
-typedef void (*RedisModuleCommandFilterFunc) (RedisModuleCtx *ctx, RedisModuleFilteredCommand *cmd);
+typedef void (*RedisModuleCommandFilterFunc) (RedisModuleCtx *ctx, RedisModuleCommandFilterCtx *filter);
 
 #define REDISMODULE_TYPE_METHOD_VERSION 1
 typedef struct RedisModuleTypeMethods {
@@ -344,6 +340,11 @@ void REDISMODULE_API_FUNC(RedisModule_SetClusterFlags)(RedisModuleCtx *ctx, uint
 int REDISMODULE_API_FUNC(RedisModule_ExportSharedAPI)(RedisModuleCtx *ctx, const char *apiname, void *func);
 void *REDISMODULE_API_FUNC(RedisModule_GetSharedAPI)(RedisModuleCtx *ctx, const char *apiname);
 int REDISMODULE_API_FUNC(RedisModule_RegisterCommandFilter)(RedisModuleCtx *ctx, RedisModuleCommandFilterFunc cb);
+int REDISMODULE_API_FUNC(RedisModule_CommandFilterArgsCount)(RedisModuleCommandFilterCtx *filter);
+const RedisModuleString *REDISMODULE_API_FUNC(RedisModule_CommandFilterArgGet)(RedisModuleCommandFilterCtx *filter, int pos);
+int REDISMODULE_API_FUNC(RedisModule_CommandFilterArgInsert)(RedisModuleCommandFilterCtx *filter, int pos, RedisModuleString *arg);
+int REDISMODULE_API_FUNC(RedisModule_CommandFilterArgReplace)(RedisModuleCommandFilterCtx *filter, int pos, RedisModuleString *arg);
+int REDISMODULE_API_FUNC(RedisModule_CommandFilterArgDelete)(RedisModuleCommandFilterCtx *filter, int pos);
 #endif
 
 /* This is included inline inside each Redis module. */
@@ -507,6 +508,11 @@ static int RedisModule_Init(RedisModuleCtx *ctx, const char *name, int ver, int
     REDISMODULE_GET_API(ExportSharedAPI);
     REDISMODULE_GET_API(GetSharedAPI);
     REDISMODULE_GET_API(RegisterCommandFilter);
+    REDISMODULE_GET_API(CommandFilterArgsCount);
+    REDISMODULE_GET_API(CommandFilterArgGet);
+    REDISMODULE_GET_API(CommandFilterArgInsert);
+    REDISMODULE_GET_API(CommandFilterArgReplace);
+    REDISMODULE_GET_API(CommandFilterArgDelete);
 #endif
 
     if (RedisModule_IsModuleNameBusy && RedisModule_IsModuleNameBusy(name)) return REDISMODULE_ERR;

From 95881cec60b54ab00b68db306a9b0078aefd6036 Mon Sep 17 00:00:00 2001
From: Yossi Gottlieb <yossigo@gmail.com>
Date: Mon, 18 Mar 2019 19:34:52 +0200
Subject: [PATCH 33/60] Add command filter Module API tests.

---
 tests/modules/commandfilter.tcl | 27 +++++++++++++++++++++++++++
 tests/test_helper.tcl           |  1 +
 2 files changed, 28 insertions(+)
 create mode 100644 tests/modules/commandfilter.tcl

diff --git a/tests/modules/commandfilter.tcl b/tests/modules/commandfilter.tcl
new file mode 100644
index 000000000..f0d96b259
--- /dev/null
+++ b/tests/modules/commandfilter.tcl
@@ -0,0 +1,27 @@
+set testmodule [file normalize src/modules/hellofilter.so]
+
+start_server {tags {"modules"}} {
+    r module load $testmodule log-key
+
+    test {Command Filter handles redirected commands} {
+        r set mykey @log
+        r lrange log-key 0 -1
+    } "{hellofilter.log set mykey @log}"
+
+    test {Command Filter can call RedisModule_CommandFilterArgDelete} {
+        r rpush mylist elem1 @delme elem2
+        r lrange mylist 0 -1
+    } {elem1 elem2}
+
+    test {Command Filter can call RedisModule_CommandFilterArgInsert} {
+        r del mylist
+        r rpush mylist elem1 @insertbefore elem2 @insertafter elem3
+        r lrange mylist 0 -1
+    } {elem1 --inserted-before-- @insertbefore elem2 @insertafter --inserted-after-- elem3}
+
+    test {Command Filter can call RedisModule_CommandFilterArgReplace} {
+        r del mylist
+        r rpush mylist elem1 @replaceme elem2
+        r lrange mylist 0 -1
+    } {elem1 --replaced-- elem2}
+} 
diff --git a/tests/test_helper.tcl b/tests/test_helper.tcl
index 568eacdee..d2f281526 100644
--- a/tests/test_helper.tcl
+++ b/tests/test_helper.tcl
@@ -63,6 +63,7 @@ set ::all_tests {
     unit/lazyfree
     unit/wait
     unit/pendingquerybuf
+    modules/commandfilter
 }
 # Index to the next test to run in the ::all_tests list.
 set ::next_test 0

From fdacd1b0b5a3a9ad1bd3ba5c1e93c51fa31b74a9 Mon Sep 17 00:00:00 2001
From: Yossi Gottlieb <yossigo@gmail.com>
Date: Mon, 18 Mar 2019 23:05:52 +0200
Subject: [PATCH 34/60] CommandFilter API: More cleanup.

---
 src/module.c      | 37 +++++++++----------------------------
 src/redismodule.h |  2 +-
 2 files changed, 10 insertions(+), 29 deletions(-)

diff --git a/src/module.c b/src/module.c
index 741c546b7..c6cb8a0ca 100644
--- a/src/module.c
+++ b/src/module.c
@@ -270,32 +270,23 @@ typedef struct RedisModuleDictIter {
     raxIterator ri;
 } RedisModuleDictIter;
 
-/* Information about the command to be executed, as passed to and from a
- * filter. */
-typedef struct RedisModuleFilteredCommand {
+typedef struct RedisModuleCommandFilterCtx {
     RedisModuleString **argv;
     int argc;
-} RedisModuleFilteredCommand;
+} RedisModuleCommandFilterCtx;
 
-typedef void (*RedisModuleCommandFilterFunc) (RedisModuleCtx *ctx, RedisModuleFilteredCommand *cmd);
+typedef void (*RedisModuleCommandFilterFunc) (RedisModuleCommandFilterCtx *filter);
 
 typedef struct RedisModuleCommandFilter {
     /* The module that registered the filter */
     RedisModule *module;
     /* Filter callback function */
     RedisModuleCommandFilterFunc callback;
-    /* Indicates a filter is active, avoid reentrancy */
-    int active;
 } RedisModuleCommandFilter;
 
 /* Registered filters */
 static list *moduleCommandFilters;
 
-typedef struct RedisModuleCommandFilterCtx {
-    RedisModuleString **argv;
-    int argc;
-} RedisModuleCommandFilterCtx;
-
 /* --------------------------------------------------------------------------
  * Prototypes
  * -------------------------------------------------------------------------- */
@@ -4802,16 +4793,13 @@ int moduleUnregisterUsedAPI(RedisModule *module) {
 
 /* Register a new command filter function.  Filters get executed by Redis
  * before processing an inbound command and can be used to manipulate the
- * behavior of standard Redis commands.  Filters must not attempt to
- * perform Redis commands or operate on the dataset, and must restrict
- * themselves to manipulation of the arguments.
+ * behavior of standard Redis commands.
  */
 
 int RM_RegisterCommandFilter(RedisModuleCtx *ctx, RedisModuleCommandFilterFunc callback) {
     RedisModuleCommandFilter *filter = zmalloc(sizeof(*filter));
     filter->module = ctx->module;
     filter->callback = callback;
-    filter->active = 0;
 
     listAddNodeTail(moduleCommandFilters, filter);
     return REDISMODULE_OK;
@@ -4824,26 +4812,19 @@ void moduleCallCommandFilters(client *c) {
     listNode *ln;
     listRewind(moduleCommandFilters,&li);
 
-    RedisModuleFilteredCommand cmd = {
+    RedisModuleCommandFilterCtx filter = {
         .argv = c->argv,
         .argc = c->argc
     };
 
     while((ln = listNext(&li))) {
-        RedisModuleCommandFilter *filter = ln->value;
-        if (filter->active) continue;
+        RedisModuleCommandFilter *f = ln->value;
 
-        RedisModuleCtx ctx = REDISMODULE_CTX_INIT;
-        ctx.module = filter->module;
-
-        filter->active = 1;
-        filter->callback(&ctx, &cmd);
-        filter->active = 0;
-        moduleFreeContext(&ctx);
+        f->callback(&filter);
     }
 
-    c->argv = cmd.argv;
-    c->argc = cmd.argc;
+    c->argv = filter.argv;
+    c->argc = filter.argc;
 }
 
 /* Return the number of arguments a filtered command has.  The number of
diff --git a/src/redismodule.h b/src/redismodule.h
index 426a6df69..5df83ae6a 100644
--- a/src/redismodule.h
+++ b/src/redismodule.h
@@ -163,7 +163,7 @@ typedef void (*RedisModuleTypeDigestFunc)(RedisModuleDigest *digest, void *value
 typedef void (*RedisModuleTypeFreeFunc)(void *value);
 typedef void (*RedisModuleClusterMessageReceiver)(RedisModuleCtx *ctx, const char *sender_id, uint8_t type, const unsigned char *payload, uint32_t len);
 typedef void (*RedisModuleTimerProc)(RedisModuleCtx *ctx, void *data);
-typedef void (*RedisModuleCommandFilterFunc) (RedisModuleCtx *ctx, RedisModuleCommandFilterCtx *filter);
+typedef void (*RedisModuleCommandFilterFunc) (RedisModuleCommandFilterCtx *filter);
 
 #define REDISMODULE_TYPE_METHOD_VERSION 1
 typedef struct RedisModuleTypeMethods {

From 5bd8aae664437cace61a422434978d8bb3740ed1 Mon Sep 17 00:00:00 2001
From: Yossi Gottlieb <yossigo@gmail.com>
Date: Mon, 18 Mar 2019 23:06:38 +0200
Subject: [PATCH 35/60] CommandFilter API: Support Lua and RM_call() flows.

---
 src/module.c    | 20 +++++++++++++-------
 src/scripting.c |  5 +++++
 2 files changed, 18 insertions(+), 7 deletions(-)

diff --git a/src/module.c b/src/module.c
index c6cb8a0ca..17accfb70 100644
--- a/src/module.c
+++ b/src/module.c
@@ -2741,12 +2741,6 @@ RedisModuleCallReply *RM_Call(RedisModuleCtx *ctx, const char *cmdname, const ch
     RedisModuleCallReply *reply = NULL;
     int replicate = 0; /* Replicate this command? */
 
-    cmd = lookupCommandByCString((char*)cmdname);
-    if (!cmd) {
-        errno = EINVAL;
-        return NULL;
-    }
-
     /* Create the client and dispatch the command. */
     va_start(ap, fmt);
     c = createClient(-1);
@@ -2760,11 +2754,23 @@ RedisModuleCallReply *RM_Call(RedisModuleCtx *ctx, const char *cmdname, const ch
     c->db = ctx->client->db;
     c->argv = argv;
     c->argc = argc;
-    c->cmd = c->lastcmd = cmd;
     /* We handle the above format error only when the client is setup so that
      * we can free it normally. */
     if (argv == NULL) goto cleanup;
 
+    /* Call command filters */
+    moduleCallCommandFilters(c);
+
+    /* Lookup command now, after filters had a chance to make modifications
+     * if necessary.
+     */
+    cmd = lookupCommand(c->argv[0]->ptr);
+    if (!cmd) {
+        errno = EINVAL;
+        goto cleanup;
+    }
+    c->cmd = c->lastcmd = cmd;
+
     /* Basic arity checks. */
     if ((cmd->arity > 0 && cmd->arity != argc) || (argc < -cmd->arity)) {
         errno = EINVAL;
diff --git a/src/scripting.c b/src/scripting.c
index cbbf43fb1..032bfdf10 100644
--- a/src/scripting.c
+++ b/src/scripting.c
@@ -462,6 +462,11 @@ int luaRedisGenericCommand(lua_State *lua, int raise_error) {
     c->argc = argc;
     c->user = server.lua_caller->user;
 
+    /* Process module hooks */
+    moduleCallCommandFilters(c);
+    argv = c->argv;
+    argc = c->argc;
+
     /* Log the command if debugging is active. */
     if (ldb.active && ldb.step) {
         sds cmdlog = sdsnew("<redis>");

From 06a6d70ab59af885c4cbfffdc023a1ad05bfa5df Mon Sep 17 00:00:00 2001
From: Yossi Gottlieb <yossigo@gmail.com>
Date: Mon, 18 Mar 2019 23:07:28 +0200
Subject: [PATCH 36/60] CommandFilter API: hellofilter and tests.

---
 src/modules/hellofilter.c       | 32 ++++++++++++++++++++++++++++----
 tests/modules/commandfilter.tcl | 20 +++++++++++++++++++-
 2 files changed, 47 insertions(+), 5 deletions(-)

diff --git a/src/modules/hellofilter.c b/src/modules/hellofilter.c
index 84eb02c30..d5dd405aa 100644
--- a/src/modules/hellofilter.c
+++ b/src/modules/hellofilter.c
@@ -6,17 +6,32 @@
 static RedisModuleString *log_key_name;
 
 static const char log_command_name[] = "hellofilter.log";
+static const char ping_command_name[] = "hellofilter.ping";
+static int in_module = 0;
+
+int HelloFilter_PingCommand(RedisModuleCtx *ctx, RedisModuleString **argv, int argc)
+{
+    RedisModuleCallReply *reply = RedisModule_Call(ctx, "ping", "c", "@log");
+    if (reply) {
+        RedisModule_ReplyWithCallReply(ctx, reply);
+        RedisModule_FreeCallReply(reply);
+    } else {
+        RedisModule_ReplyWithSimpleString(ctx, "Unknown command or invalid arguments");
+    }
+
+    return REDISMODULE_OK;
+}
 
 int HelloFilter_LogCommand(RedisModuleCtx *ctx, RedisModuleString **argv, int argc)
 {
-    RedisModuleString *s = RedisModule_CreateStringFromString(ctx, argv[0]);
+    RedisModuleString *s = RedisModule_CreateString(ctx, "", 0);
 
     int i;
     for (i = 1; i < argc; i++) {
         size_t arglen;
         const char *arg = RedisModule_StringPtrLen(argv[i], &arglen);
 
-        RedisModule_StringAppendBuffer(ctx, s, " ", 1);
+        if (i > 1) RedisModule_StringAppendBuffer(ctx, s, " ", 1);
         RedisModule_StringAppendBuffer(ctx, s, arg, arglen);
     }
 
@@ -25,6 +40,8 @@ int HelloFilter_LogCommand(RedisModuleCtx *ctx, RedisModuleString **argv, int ar
     RedisModule_CloseKey(log);
     RedisModule_FreeString(ctx, s);
 
+    in_module = 1;
+
     size_t cmdlen;
     const char *cmdname = RedisModule_StringPtrLen(argv[1], &cmdlen);
     RedisModuleCallReply *reply = RedisModule_Call(ctx, cmdname, "v", &argv[2], argc - 2);
@@ -34,12 +51,15 @@ int HelloFilter_LogCommand(RedisModuleCtx *ctx, RedisModuleString **argv, int ar
     } else {
         RedisModule_ReplyWithSimpleString(ctx, "Unknown command or invalid arguments");
     }
+
+    in_module = 0;
+
     return REDISMODULE_OK;
 }
 
-void HelloFilter_CommandFilter(RedisModuleCtx *ctx, RedisModuleCommandFilterCtx *filter)
+void HelloFilter_CommandFilter(RedisModuleCommandFilterCtx *filter)
 {
-    (void) ctx;
+    if (in_module) return;  /* don't process our own RM_Call() */
 
     /* Fun manipulations:
      * - Remove @delme
@@ -94,6 +114,10 @@ int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc)
                 HelloFilter_LogCommand,"write deny-oom",1,1,1) == REDISMODULE_ERR)
             return REDISMODULE_ERR;
 
+    if (RedisModule_CreateCommand(ctx,ping_command_name,
+                HelloFilter_PingCommand,"write deny-oom",1,1,1) == REDISMODULE_ERR)
+            return REDISMODULE_ERR;
+
     if (RedisModule_RegisterCommandFilter(ctx, HelloFilter_CommandFilter)
             == REDISMODULE_ERR) return REDISMODULE_ERR;
 
diff --git a/tests/modules/commandfilter.tcl b/tests/modules/commandfilter.tcl
index f0d96b259..47d9c302c 100644
--- a/tests/modules/commandfilter.tcl
+++ b/tests/modules/commandfilter.tcl
@@ -6,7 +6,7 @@ start_server {tags {"modules"}} {
     test {Command Filter handles redirected commands} {
         r set mykey @log
         r lrange log-key 0 -1
-    } "{hellofilter.log set mykey @log}"
+    } "{set mykey @log}"
 
     test {Command Filter can call RedisModule_CommandFilterArgDelete} {
         r rpush mylist elem1 @delme elem2
@@ -24,4 +24,22 @@ start_server {tags {"modules"}} {
         r rpush mylist elem1 @replaceme elem2
         r lrange mylist 0 -1
     } {elem1 --replaced-- elem2}
+
+    test {Command Filter applies on RM_Call() commands} {
+        r del log-key
+        r hellofilter.ping
+        r lrange log-key 0 -1
+    } "{ping @log}"
+
+    test {Command Filter applies on Lua redis.call()} {
+        r del log-key
+        r eval "redis.call('ping', '@log')" 0
+        r lrange log-key 0 -1
+    } "{ping @log}"
+
+    test {Command Filter applies on Lua redis.call() that calls a module} {
+        r del log-key
+        r eval "redis.call('hellofilter.ping')" 0
+        r lrange log-key 0 -1
+    } "{ping @log}"
 } 

From 278c7a6b6d679dfda65a35c098e1fd763a974b86 Mon Sep 17 00:00:00 2001
From: Dvir Volk <dvirsky@google.com>
Date: Tue, 19 Mar 2019 13:11:37 +0200
Subject: [PATCH 37/60] Added keyspace miss notifications support

---
 src/db.c                 |  7 ++++++-
 src/modules/testmodule.c | 29 ++++++++++++++++++++++-------
 2 files changed, 28 insertions(+), 8 deletions(-)

diff --git a/src/db.c b/src/db.c
index 7950d5074..8c9047084 100644
--- a/src/db.c
+++ b/src/db.c
@@ -83,6 +83,7 @@ robj *lookupKey(redisDb *db, robj *key, int flags) {
  * 1. A key gets expired if it reached it's TTL.
  * 2. The key last access time is updated.
  * 3. The global keys hits/misses stats are updated (reported in INFO).
+ * 4. If keyspace notifications are enabled, a "miss" notification is fired.
  *
  * This API should not be used when we write to the key after obtaining
  * the object linked to the key, but only for read only operations.
@@ -106,6 +107,7 @@ robj *lookupKeyReadWithFlags(redisDb *db, robj *key, int flags) {
          * to return NULL ASAP. */
         if (server.masterhost == NULL) {
             server.stat_keyspace_misses++;
+            notifyKeyspaceEvent(NOTIFY_GENERIC, "miss", key, db->id);
             return NULL;
         }
 
@@ -127,12 +129,15 @@ robj *lookupKeyReadWithFlags(redisDb *db, robj *key, int flags) {
             server.current_client->cmd->flags & CMD_READONLY)
         {
             server.stat_keyspace_misses++;
+            notifyKeyspaceEvent(NOTIFY_GENERIC, "miss", key, db->id);
             return NULL;
         }
     }
     val = lookupKey(db,key,flags);
-    if (val == NULL)
+    if (val == NULL) {
         server.stat_keyspace_misses++;
+        notifyKeyspaceEvent(NOTIFY_GENERIC, "miss", key, db->id);
+    }
     else
         server.stat_keyspace_hits++;
     return val;
diff --git a/src/modules/testmodule.c b/src/modules/testmodule.c
index 67a861704..826dd9a7e 100644
--- a/src/modules/testmodule.c
+++ b/src/modules/testmodule.c
@@ -109,9 +109,9 @@ int TestStringPrintf(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
     if (argc < 3) {
         return RedisModule_WrongArity(ctx);
     }
-    RedisModuleString *s = RedisModule_CreateStringPrintf(ctx, 
-        "Got %d args. argv[1]: %s, argv[2]: %s", 
-        argc, 
+    RedisModuleString *s = RedisModule_CreateStringPrintf(ctx,
+        "Got %d args. argv[1]: %s, argv[2]: %s",
+        argc,
         RedisModule_StringPtrLen(argv[1], NULL),
         RedisModule_StringPtrLen(argv[2], NULL)
     );
@@ -133,7 +133,7 @@ int TestUnlink(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
 
     RedisModuleKey *k = RedisModule_OpenKey(ctx, RedisModule_CreateStringPrintf(ctx, "unlinked"), REDISMODULE_WRITE | REDISMODULE_READ);
     if (!k) return failTest(ctx, "Could not create key");
-    
+
     if (REDISMODULE_ERR == RedisModule_StringSet(k, RedisModule_CreateStringPrintf(ctx, "Foobar"))) {
         return failTest(ctx, "Could not set string value");
     }
@@ -152,7 +152,7 @@ int TestUnlink(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
         return failTest(ctx, "Could not verify key to be unlinked");
     }
     return RedisModule_ReplyWithSimpleString(ctx, "OK");
-    
+
 }
 
 int NotifyCallback(RedisModuleCtx *ctx, int type, const char *event,
@@ -188,6 +188,10 @@ int TestNotifications(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
     RedisModule_Call(ctx, "LPUSH", "cc", "l", "y");
     RedisModule_Call(ctx, "LPUSH", "cc", "l", "y");
 
+    /* Miss some keys intentionally so we will get a "miss" notification. */
+    RedisModule_Call(ctx, "GET", "c", "nosuchkey");
+    RedisModule_Call(ctx, "SMEMBERS", "c", "nosuchkey");
+
     size_t sz;
     const char *rep;
     RedisModuleCallReply *r = RedisModule_Call(ctx, "HGET", "cc", "notifications", "foo");
@@ -225,6 +229,16 @@ int TestNotifications(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
         FAIL("Wrong reply for l");
     }
 
+    r = RedisModule_Call(ctx, "HGET", "cc", "notifications", "nosuchkey");
+    if (r == NULL || RedisModule_CallReplyType(r) != REDISMODULE_REPLY_STRING) {
+        FAIL("Wrong or no reply for nosuchkey");
+    } else {
+        rep = RedisModule_CallReplyStringPtr(r, &sz);
+        if (sz != 1 || *rep != '2') {
+            FAIL("Got reply '%.*s'. expected '2'", sz, rep);
+        }
+    }
+
     RedisModule_Call(ctx, "FLUSHDB", "");
 
     return RedisModule_ReplyWithSimpleString(ctx, "OK");
@@ -423,7 +437,7 @@ int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc)
     if (RedisModule_CreateCommand(ctx,"test.ctxflags",
         TestCtxFlags,"readonly",1,1,1) == REDISMODULE_ERR)
         return REDISMODULE_ERR;
-    
+
     if (RedisModule_CreateCommand(ctx,"test.unlink",
         TestUnlink,"write deny-oom",1,1,1) == REDISMODULE_ERR)
         return REDISMODULE_ERR;
@@ -435,7 +449,8 @@ int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc)
     RedisModule_SubscribeToKeyspaceEvents(ctx,
                                             REDISMODULE_NOTIFY_HASH |
                                             REDISMODULE_NOTIFY_SET |
-                                            REDISMODULE_NOTIFY_STRING,
+                                            REDISMODULE_NOTIFY_STRING |
+                                            REDISMODULE_NOTIFY_GENERIC,
                                         NotifyCallback);
     if (RedisModule_CreateCommand(ctx,"test.notify",
         TestNotifications,"write deny-oom",1,1,1) == REDISMODULE_ERR)

From 1da0d9b04cded3f65ca7c6aecbee07c02d118696 Mon Sep 17 00:00:00 2001
From: Yossi Gottlieb <yossigo@gmail.com>
Date: Tue, 19 Mar 2019 19:48:47 +0200
Subject: [PATCH 38/60] CommandFilter API: Extend documentation.

---
 src/module.c | 48 +++++++++++++++++++++++++++++++++++++++++++-----
 1 file changed, 43 insertions(+), 5 deletions(-)

diff --git a/src/module.c b/src/module.c
index 17accfb70..ee2840225 100644
--- a/src/module.c
+++ b/src/module.c
@@ -4797,9 +4797,47 @@ int moduleUnregisterUsedAPI(RedisModule *module) {
  * Module Command Filter API
  * -------------------------------------------------------------------------- */
 
-/* Register a new command filter function.  Filters get executed by Redis
- * before processing an inbound command and can be used to manipulate the
- * behavior of standard Redis commands.
+/* Register a new command filter function.
+ *
+ * Command filtering makes it possible for modules to extend Redis by plugging
+ * into the execution flow of all commands.
+ *
+ * A registered filter gets called before Redis executes *any* command.  This
+ * includes both core Redis commands and commands registered by any module.  The
+ * filter applies in all execution paths including:
+ *
+ * 1. Invocation by a client.
+ * 2. Invocation through `RedisModule_Call()` by any module.
+ * 3. Invocation through Lua 'redis.call()`.
+ * 4. Replication of a command from a master.
+ *
+ * The filter executes in a special filter context, which is different and more
+ * limited than a RedisModuleCtx.  Because the filter affects any command, it
+ * must be implemented in a very efficient way to reduce the performance impact
+ * on Redis.  All Redis Module API calls that require a valid context (such as
+ * `RedisModule_Call()`, `RedisModule_OpenKey()`, etc.) are not supported in a
+ * filter context.
+ *
+ * The `RedisModuleCommandFilterCtx` can be used to inspect or modify the
+ * executed command and its arguments.  As the filter executes before Redis
+ * begins processing the command, any change will affect the way the command is
+ * processed.  For example, a module can override Redis commands this way:
+ *
+ * 1. Register a `MODULE.SET` command which implements an extended version of
+ *    the Redis `SET` command.
+ * 2. Register a command filter which detects invocation of `SET` on a specific
+ *    pattern of keys.  Once detected, the filter will replace the first
+ *    argument from `SET` to `MODULE.SET`.
+ * 3. When filter execution is complete, Redis considers the new command name
+ *    and therefore executes the module's own command.
+ *
+ * Note that in the above use case, if `MODULE.SET` itself uses
+ * `RedisModule_Call()` the filter will be applied on that call as well.  If
+ * that is not desired, the module itself is responsible for maintaining a flag
+ * to identify and avoid this form of re-entrancy.
+ *
+ * If multiple filters are registered (by the same or different modules), they
+ * are executed in the order of registration.
  */
 
 int RM_RegisterCommandFilter(RedisModuleCtx *ctx, RedisModuleCommandFilterFunc callback) {
@@ -4881,7 +4919,7 @@ int RM_CommandFilterArgInsert(RedisModuleCommandFilterCtx *filter, int pos, Redi
 int RM_CommandFilterArgReplace(RedisModuleCommandFilterCtx *filter, int pos, RedisModuleString *arg)
 {
     if (pos < 0 || pos >= filter->argc) return REDISMODULE_ERR;
-    
+
     decrRefCount(filter->argv[pos]);
     filter->argv[pos] = arg;
 
@@ -4901,7 +4939,7 @@ int RM_CommandFilterArgDelete(RedisModuleCommandFilterCtx *filter, int pos)
         filter->argv[i] = filter->argv[i+1];
     }
     filter->argc--;
-    
+
     return REDISMODULE_OK;
 }
 

From 61501773c977fc4e486669304705ebd4e73ac716 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Thu, 27 Sep 2018 18:12:31 +0300
Subject: [PATCH 39/60] getKeysFromCommand for TOUCH only extracted the first
 key.

also, airty for COMMAND command was wrong.
---
 src/server.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/server.c b/src/server.c
index 712cda1bd..1341ab405 100644
--- a/src/server.c
+++ b/src/server.c
@@ -715,7 +715,7 @@ struct redisCommand redisCommandTable[] = {
 
     {"touch",touchCommand,-2,
      "read-only fast @keyspace",
-     0,NULL,1,1,1,0,0,0},
+     0,NULL,1,-1,1,0,0,0},
 
     {"pttl",pttlCommand,2,
      "read-only fast random @keyspace",
@@ -863,7 +863,7 @@ struct redisCommand redisCommandTable[] = {
      "no-script @keyspace",
      0,NULL,0,0,0,0,0,0},
 
-    {"command",commandCommand,0,
+    {"command",commandCommand,-1,
      "ok-loading ok-stale random @connection",
      0,NULL,0,0,0,0,0,0},
 

From 9b2a42636f49b484f7ec62fccb8e6ed820f86e42 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Thu, 27 Sep 2018 18:03:47 +0300
Subject: [PATCH 40/60] change SORT and SPOP to use lookupKeyWrite rather than
 lookupKeyRead

like in SUNIONSTORE etc, commands that perform writes are expected to open
all keys, even input keys, with lookupKeyWrite
---
 src/sort.c  | 55 ++++++++++++++++++++++++++++++-----------------------
 src/t_set.c |  2 +-
 2 files changed, 32 insertions(+), 25 deletions(-)

diff --git a/src/sort.c b/src/sort.c
index 8608cd8b3..db26da158 100644
--- a/src/sort.c
+++ b/src/sort.c
@@ -58,7 +58,7 @@ redisSortOperation *createSortOperation(int type, robj *pattern) {
  *
  * The returned object will always have its refcount increased by 1
  * when it is non-NULL. */
-robj *lookupKeyByPattern(redisDb *db, robj *pattern, robj *subst) {
+robj *lookupKeyByPattern(redisDb *db, robj *pattern, robj *subst, int writeflag) {
     char *p, *f, *k;
     sds spat, ssub;
     robj *keyobj, *fieldobj = NULL, *o;
@@ -106,7 +106,10 @@ robj *lookupKeyByPattern(redisDb *db, robj *pattern, robj *subst) {
     decrRefCount(subst); /* Incremented by decodeObject() */
 
     /* Lookup substituted key */
-    o = lookupKeyRead(db,keyobj);
+    if (!writeflag)
+        o = lookupKeyRead(db,keyobj);
+    else
+        o = lookupKeyWrite(db,keyobj);
     if (o == NULL) goto noobj;
 
     if (fieldobj) {
@@ -198,30 +201,12 @@ void sortCommand(client *c) {
     robj *sortval, *sortby = NULL, *storekey = NULL;
     redisSortObject *vector; /* Resulting vector to sort */
 
-    /* Lookup the key to sort. It must be of the right types */
-    sortval = lookupKeyRead(c->db,c->argv[1]);
-    if (sortval && sortval->type != OBJ_SET &&
-                   sortval->type != OBJ_LIST &&
-                   sortval->type != OBJ_ZSET)
-    {
-        addReply(c,shared.wrongtypeerr);
-        return;
-    }
-
     /* Create a list of operations to perform for every sorted element.
      * Operations can be GET */
     operations = listCreate();
     listSetFreeMethod(operations,zfree);
     j = 2; /* options start at argv[2] */
 
-    /* Now we need to protect sortval incrementing its count, in the future
-     * SORT may have options able to overwrite/delete keys during the sorting
-     * and the sorted key itself may get destroyed */
-    if (sortval)
-        incrRefCount(sortval);
-    else
-        sortval = createQuicklistObject();
-
     /* The SORT command has an SQL-alike syntax, parse it */
     while(j < c->argc) {
         int leftargs = c->argc-j-1;
@@ -280,11 +265,33 @@ void sortCommand(client *c) {
 
     /* Handle syntax errors set during options parsing. */
     if (syntax_error) {
-        decrRefCount(sortval);
         listRelease(operations);
         return;
     }
 
+    /* Lookup the key to sort. It must be of the right types */
+    if (storekey)
+        sortval = lookupKeyRead(c->db,c->argv[1]);
+    else
+        sortval = lookupKeyWrite(c->db,c->argv[1]);
+    if (sortval && sortval->type != OBJ_SET &&
+                   sortval->type != OBJ_LIST &&
+                   sortval->type != OBJ_ZSET)
+    {
+        listRelease(operations);
+        addReply(c,shared.wrongtypeerr);
+        return;
+    }
+
+    /* Now we need to protect sortval incrementing its count, in the future
+     * SORT may have options able to overwrite/delete keys during the sorting
+     * and the sorted key itself may get destroyed */
+    if (sortval)
+        incrRefCount(sortval);
+    else
+        sortval = createQuicklistObject();
+
+
     /* When sorting a set with no sort specified, we must sort the output
      * so the result is consistent across scripting and replication.
      *
@@ -452,7 +459,7 @@ void sortCommand(client *c) {
             robj *byval;
             if (sortby) {
                 /* lookup value to sort by */
-                byval = lookupKeyByPattern(c->db,sortby,vector[j].obj);
+                byval = lookupKeyByPattern(c->db,sortby,vector[j].obj,storekey!=NULL);
                 if (!byval) continue;
             } else {
                 /* use object itself to sort by */
@@ -515,7 +522,7 @@ void sortCommand(client *c) {
             while((ln = listNext(&li))) {
                 redisSortOperation *sop = ln->value;
                 robj *val = lookupKeyByPattern(c->db,sop->pattern,
-                    vector[j].obj);
+                    vector[j].obj,storekey!=NULL);
 
                 if (sop->type == SORT_OP_GET) {
                     if (!val) {
@@ -545,7 +552,7 @@ void sortCommand(client *c) {
                 while((ln = listNext(&li))) {
                     redisSortOperation *sop = ln->value;
                     robj *val = lookupKeyByPattern(c->db,sop->pattern,
-                        vector[j].obj);
+                        vector[j].obj,storekey!=NULL);
 
                     if (sop->type == SORT_OP_GET) {
                         if (!val) val = createStringObject("",0);
diff --git a/src/t_set.c b/src/t_set.c
index cbe55aaa4..05d9ee243 100644
--- a/src/t_set.c
+++ b/src/t_set.c
@@ -415,7 +415,7 @@ void spopWithCountCommand(client *c) {
 
     /* Make sure a key with the name inputted exists, and that it's type is
      * indeed a set. Otherwise, return nil */
-    if ((set = lookupKeyReadOrReply(c,c->argv[1],shared.null[c->resp]))
+    if ((set = lookupKeyWriteOrReply(c,c->argv[1],shared.null[c->resp]))
         == NULL || checkType(c,set,OBJ_SET)) return;
 
     /* If count is zero, serve an empty multibulk ASAP to avoid special

From 4355e2974955c5d39b47df9c9f2c1e81b09fc9b9 Mon Sep 17 00:00:00 2001
From: oranagra <oran@redislabs.com>
Date: Thu, 23 Feb 2017 03:13:44 -0800
Subject: [PATCH 41/60] bugfix to restartAOF, exit will never happen since
 retry will get negative.

also reduce an excess sleep
---
 src/replication.c | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/src/replication.c b/src/replication.c
index f2adc7995..59e42e561 100644
--- a/src/replication.c
+++ b/src/replication.c
@@ -1091,12 +1091,13 @@ void replicationCreateMasterClient(int fd, int dbid) {
 }
 
 void restartAOF() {
-    int retry = 10;
-    while (retry-- && startAppendOnly() == C_ERR) {
+    unsigned int tries, max_tries = 10;
+    for (tries = 0; tries < max_tries; ++tries) {
+        if (tries) sleep(1);
+        if (startAppendOnly() == C_OK) break;
         serverLog(LL_WARNING,"Failed enabling the AOF after successful master synchronization! Trying it again in one second.");
-        sleep(1);
     }
-    if (!retry) {
+    if (tries == max_tries) {
         serverLog(LL_WARNING,"FATAL: this replica instance finished the synchronization with its master, but the AOF can't be turned on. Exiting now.");
         exit(1);
     }

From 544b9b08265fbb1696ec4a27a4c09035cd099a94 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Wed, 20 Mar 2019 17:46:19 +0200
Subject: [PATCH 42/60] diskless replication - notify slave when rdb transfer
 failed

in diskless replication - master was not notifing the slave that rdb transfer
terminated on error, and lets slave wait for replication timeout
---
 src/replication.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/replication.c b/src/replication.c
index f2adc7995..8f0d67914 100644
--- a/src/replication.c
+++ b/src/replication.c
@@ -593,6 +593,7 @@ int startBgsaveForReplication(int mincapa) {
             client *slave = ln->value;
 
             if (slave->replstate == SLAVE_STATE_WAIT_BGSAVE_START) {
+                slave->replstate = REPL_STATE_NONE;
                 slave->flags &= ~CLIENT_SLAVE;
                 listDelNode(server.slaves,ln);
                 addReplyError(slave,

From 50befc42ad1523942b82af81e6722b66744825f9 Mon Sep 17 00:00:00 2001
From: Dvir Volk <dvirsky@google.com>
Date: Thu, 21 Mar 2019 11:47:14 +0200
Subject: [PATCH 43/60] added special flag for keyspace miss notifications

---
 src/db.c                 | 6 +++---
 src/modules/testmodule.c | 2 +-
 src/notify.c             | 6 ++++--
 src/redismodule.h        | 1 +
 src/server.h             | 4 +++-
 5 files changed, 12 insertions(+), 7 deletions(-)

diff --git a/src/db.c b/src/db.c
index 8c9047084..afe181281 100644
--- a/src/db.c
+++ b/src/db.c
@@ -107,7 +107,7 @@ robj *lookupKeyReadWithFlags(redisDb *db, robj *key, int flags) {
          * to return NULL ASAP. */
         if (server.masterhost == NULL) {
             server.stat_keyspace_misses++;
-            notifyKeyspaceEvent(NOTIFY_GENERIC, "miss", key, db->id);
+            notifyKeyspaceEvent(NOTIFY_KEY_MISS, "miss", key, db->id);
             return NULL;
         }
 
@@ -129,14 +129,14 @@ robj *lookupKeyReadWithFlags(redisDb *db, robj *key, int flags) {
             server.current_client->cmd->flags & CMD_READONLY)
         {
             server.stat_keyspace_misses++;
-            notifyKeyspaceEvent(NOTIFY_GENERIC, "miss", key, db->id);
+            notifyKeyspaceEvent(NOTIFY_KEY_MISS, "miss", key, db->id);
             return NULL;
         }
     }
     val = lookupKey(db,key,flags);
     if (val == NULL) {
         server.stat_keyspace_misses++;
-        notifyKeyspaceEvent(NOTIFY_GENERIC, "miss", key, db->id);
+        notifyKeyspaceEvent(NOTIFY_KEY_MISS, "miss", key, db->id);
     }
     else
         server.stat_keyspace_hits++;
diff --git a/src/modules/testmodule.c b/src/modules/testmodule.c
index 826dd9a7e..af78d21d7 100644
--- a/src/modules/testmodule.c
+++ b/src/modules/testmodule.c
@@ -450,7 +450,7 @@ int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc)
                                             REDISMODULE_NOTIFY_HASH |
                                             REDISMODULE_NOTIFY_SET |
                                             REDISMODULE_NOTIFY_STRING |
-                                            REDISMODULE_NOTIFY_GENERIC,
+                                            REDISMODULE_NOTIFY_KEY_MISS,
                                         NotifyCallback);
     if (RedisModule_CreateCommand(ctx,"test.notify",
         TestNotifications,"write deny-oom",1,1,1) == REDISMODULE_ERR)
diff --git a/src/notify.c b/src/notify.c
index 1afb36fc0..d6c3ad403 100644
--- a/src/notify.c
+++ b/src/notify.c
@@ -55,6 +55,7 @@ int keyspaceEventsStringToFlags(char *classes) {
         case 'K': flags |= NOTIFY_KEYSPACE; break;
         case 'E': flags |= NOTIFY_KEYEVENT; break;
         case 't': flags |= NOTIFY_STREAM; break;
+        case 'm': flags |= NOTIFY_KEY_MISS; break;
         default: return -1;
         }
     }
@@ -81,6 +82,7 @@ sds keyspaceEventsFlagsToString(int flags) {
         if (flags & NOTIFY_EXPIRED) res = sdscatlen(res,"x",1);
         if (flags & NOTIFY_EVICTED) res = sdscatlen(res,"e",1);
         if (flags & NOTIFY_STREAM) res = sdscatlen(res,"t",1);
+        if (flags & NOTIFY_KEY_MISS) res = sdscatlen(res,"m",1);
     }
     if (flags & NOTIFY_KEYSPACE) res = sdscatlen(res,"K",1);
     if (flags & NOTIFY_KEYEVENT) res = sdscatlen(res,"E",1);
@@ -100,12 +102,12 @@ void notifyKeyspaceEvent(int type, char *event, robj *key, int dbid) {
     int len = -1;
     char buf[24];
 
-    /* If any modules are interested in events, notify the module system now. 
+    /* If any modules are interested in events, notify the module system now.
      * This bypasses the notifications configuration, but the module engine
      * will only call event subscribers if the event type matches the types
      * they are interested in. */
      moduleNotifyKeyspaceEvent(type, event, key, dbid);
-    
+
     /* If notifications for this class of events are off, return ASAP. */
     if (!(server.notify_keyspace_events & type)) return;
 
diff --git a/src/redismodule.h b/src/redismodule.h
index 272da08df..681bd600b 100644
--- a/src/redismodule.h
+++ b/src/redismodule.h
@@ -98,6 +98,7 @@
 #define REDISMODULE_NOTIFY_EXPIRED (1<<8)     /* x */
 #define REDISMODULE_NOTIFY_EVICTED (1<<9)     /* e */
 #define REDISMODULE_NOTIFY_STREAM (1<<10)     /* t */
+#define REDISMODULE_NOTIFY_KEY_MISS (1<<11)   /* m */
 #define REDISMODULE_NOTIFY_ALL (REDISMODULE_NOTIFY_GENERIC | REDISMODULE_NOTIFY_STRING | REDISMODULE_NOTIFY_LIST | REDISMODULE_NOTIFY_SET | REDISMODULE_NOTIFY_HASH | REDISMODULE_NOTIFY_ZSET | REDISMODULE_NOTIFY_EXPIRED | REDISMODULE_NOTIFY_EVICTED | REDISMODULE_NOTIFY_STREAM)      /* A */
 
 
diff --git a/src/server.h b/src/server.h
index 56c3b67d3..0b433039d 100644
--- a/src/server.h
+++ b/src/server.h
@@ -468,7 +468,9 @@ typedef long long mstime_t; /* millisecond time type. */
 #define NOTIFY_EXPIRED (1<<8)     /* x */
 #define NOTIFY_EVICTED (1<<9)     /* e */
 #define NOTIFY_STREAM (1<<10)     /* t */
-#define NOTIFY_ALL (NOTIFY_GENERIC | NOTIFY_STRING | NOTIFY_LIST | NOTIFY_SET | NOTIFY_HASH | NOTIFY_ZSET | NOTIFY_EXPIRED | NOTIFY_EVICTED | NOTIFY_STREAM) /* A flag */
+#define NOTIFY_KEY_MISS (1<<11)   /* m */
+
+#define NOTIFY_ALL (NOTIFY_GENERIC | NOTIFY_STRING | NOTIFY_LIST | NOTIFY_SET | NOTIFY_HASH | NOTIFY_ZSET | NOTIFY_EXPIRED | NOTIFY_EVICTED | NOTIFY_STREAM | NOTIFY_KEY_MISS) /* A flag */
 
 /* Get the first bind addr or NULL */
 #define NET_FIRST_BIND_ADDR (server.bindaddr_count ? server.bindaddr[0] : NULL)

From ca2eadaaac824d329c877820cd5d159607e0227b Mon Sep 17 00:00:00 2001
From: Dvir Volk <dvirsky@google.com>
Date: Thu, 21 Mar 2019 12:47:51 +0200
Subject: [PATCH 44/60] Added missing REDISMODULE_NOTIFY_KEY_MISS flag to
 REDISMODULE_NOTIFY_ALL

---
 src/redismodule.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/redismodule.h b/src/redismodule.h
index 681bd600b..70011f932 100644
--- a/src/redismodule.h
+++ b/src/redismodule.h
@@ -99,7 +99,7 @@
 #define REDISMODULE_NOTIFY_EVICTED (1<<9)     /* e */
 #define REDISMODULE_NOTIFY_STREAM (1<<10)     /* t */
 #define REDISMODULE_NOTIFY_KEY_MISS (1<<11)   /* m */
-#define REDISMODULE_NOTIFY_ALL (REDISMODULE_NOTIFY_GENERIC | REDISMODULE_NOTIFY_STRING | REDISMODULE_NOTIFY_LIST | REDISMODULE_NOTIFY_SET | REDISMODULE_NOTIFY_HASH | REDISMODULE_NOTIFY_ZSET | REDISMODULE_NOTIFY_EXPIRED | REDISMODULE_NOTIFY_EVICTED | REDISMODULE_NOTIFY_STREAM)      /* A */
+#define REDISMODULE_NOTIFY_ALL (REDISMODULE_NOTIFY_GENERIC | REDISMODULE_NOTIFY_STRING | REDISMODULE_NOTIFY_LIST | REDISMODULE_NOTIFY_SET | REDISMODULE_NOTIFY_HASH | REDISMODULE_NOTIFY_ZSET | REDISMODULE_NOTIFY_EXPIRED | REDISMODULE_NOTIFY_EVICTED | REDISMODULE_NOTIFY_STREAM | REDISMODULE_NOTIFY_KEY_MISS)      /* A */
 
 
 /* A special pointer that we can use between the core and the module to signal

From 51a54dfde3c27e162dfedcddb663350575cd7733 Mon Sep 17 00:00:00 2001
From: Dvir Volk <dvirsky@google.com>
Date: Thu, 21 Mar 2019 12:48:37 +0200
Subject: [PATCH 45/60] remove extra linebreak

---
 src/server.h | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/server.h b/src/server.h
index 0b433039d..b090a6374 100644
--- a/src/server.h
+++ b/src/server.h
@@ -469,7 +469,6 @@ typedef long long mstime_t; /* millisecond time type. */
 #define NOTIFY_EVICTED (1<<9)     /* e */
 #define NOTIFY_STREAM (1<<10)     /* t */
 #define NOTIFY_KEY_MISS (1<<11)   /* m */
-
 #define NOTIFY_ALL (NOTIFY_GENERIC | NOTIFY_STRING | NOTIFY_LIST | NOTIFY_SET | NOTIFY_HASH | NOTIFY_ZSET | NOTIFY_EXPIRED | NOTIFY_EVICTED | NOTIFY_STREAM | NOTIFY_KEY_MISS) /* A flag */
 
 /* Get the first bind addr or NULL */

From 8b58fbafae448f9ff106429bc5d53e1cd20bcfea Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Thu, 21 Mar 2019 12:18:55 +0100
Subject: [PATCH 46/60] Alter coding style in #4696 to conform to Redis code
 base.

---
 src/zmalloc.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/zmalloc.c b/src/zmalloc.c
index 4c40a7782..5e6010278 100644
--- a/src/zmalloc.c
+++ b/src/zmalloc.c
@@ -148,7 +148,7 @@ void *zrealloc(void *ptr, size_t size) {
     size_t oldsize;
     void *newptr;
 
-    if (size == 0 && ptr!=NULL) {
+    if (size == 0 && ptr != NULL) {
         zfree(ptr);
         return NULL;
     }

From c675d44488094bc6e360f8a330375d3b9fb3b335 Mon Sep 17 00:00:00 2001
From: Yossi Gottlieb <yossigo@gmail.com>
Date: Thu, 21 Mar 2019 14:44:49 +0200
Subject: [PATCH 47/60] CommandFilter API: Add unregister option.

A filter handle is returned and can be used to unregister a filter.  In
the future it can also be used to further configure or manipulate the
filter.

Filters are now automatically unregistered when a module unloads.
---
 src/module.c                    | 94 +++++++++++++++++++++++++--------
 src/modules/hellofilter.c       | 27 ++++++++--
 src/redismodule.h               | 15 +++---
 tests/modules/commandfilter.tcl | 22 ++++++++
 4 files changed, 126 insertions(+), 32 deletions(-)

diff --git a/src/module.c b/src/module.c
index ee2840225..ad7bba2eb 100644
--- a/src/module.c
+++ b/src/module.c
@@ -49,6 +49,7 @@ struct RedisModule {
     list *types;    /* Module data types. */
     list *usedby;   /* List of modules using APIs from this one. */
     list *using;    /* List of modules we use some APIs of. */
+    list *filters;  /* List of filters the module has registered. */
 };
 typedef struct RedisModule RedisModule;
 
@@ -748,6 +749,7 @@ void RM_SetModuleAttribs(RedisModuleCtx *ctx, const char *name, int ver, int api
     module->types = listCreate();
     module->usedby = listCreate();
     module->using = listCreate();
+    module->filters = listCreate();
     ctx->module = module;
 }
 
@@ -4793,6 +4795,28 @@ int moduleUnregisterUsedAPI(RedisModule *module) {
     return count;
 }
 
+/* Unregister all filters registered by a module.
+ * This is called when a module is being unloaded.
+ * 
+ * Returns the number of filters unregistered. */
+int moduleUnregisterFilters(RedisModule *module) {
+    listIter li;
+    listNode *ln;
+    int count = 0;
+
+    listRewind(module->filters,&li);
+    while((ln = listNext(&li))) {
+        RedisModuleCommandFilter *filter = ln->value;
+        listNode *ln = listSearchKey(moduleCommandFilters,filter);
+        if (ln) {
+            listDelNode(moduleCommandFilters,ln);
+            count++;
+        }
+        zfree(filter);
+    }
+    return count;
+}
+
 /* --------------------------------------------------------------------------
  * Module Command Filter API
  * -------------------------------------------------------------------------- */
@@ -4840,12 +4864,33 @@ int moduleUnregisterUsedAPI(RedisModule *module) {
  * are executed in the order of registration.
  */
 
-int RM_RegisterCommandFilter(RedisModuleCtx *ctx, RedisModuleCommandFilterFunc callback) {
+RedisModuleCommandFilter *RM_RegisterCommandFilter(RedisModuleCtx *ctx, RedisModuleCommandFilterFunc callback) {
     RedisModuleCommandFilter *filter = zmalloc(sizeof(*filter));
     filter->module = ctx->module;
     filter->callback = callback;
 
     listAddNodeTail(moduleCommandFilters, filter);
+    listAddNodeTail(ctx->module->filters, filter);
+    return filter;
+}
+
+/* Unregister a command filter.
+ */
+int RM_UnregisterCommandFilter(RedisModuleCtx *ctx, RedisModuleCommandFilter *filter) {
+    listNode *ln;
+
+    /* A module can only remove its own filters */
+    if (filter->module != ctx->module) return REDISMODULE_ERR;
+
+    ln = listSearchKey(moduleCommandFilters,filter);
+    if (!ln) return REDISMODULE_ERR;
+    listDelNode(moduleCommandFilters,ln);
+    
+    ln = listSearchKey(ctx->module->filters,filter);
+    if (ln) {
+        listDelNode(moduleCommandFilters,ln);
+    }
+
     return REDISMODULE_OK;
 }
 
@@ -4874,18 +4919,18 @@ void moduleCallCommandFilters(client *c) {
 /* Return the number of arguments a filtered command has.  The number of
  * arguments include the command itself.
  */
-int RM_CommandFilterArgsCount(RedisModuleCommandFilterCtx *filter)
+int RM_CommandFilterArgsCount(RedisModuleCommandFilterCtx *fctx)
 {
-    return filter->argc;
+    return fctx->argc;
 }
 
 /* Return the specified command argument.  The first argument (position 0) is
  * the command itself, and the rest are user-provided args.
  */
-const RedisModuleString *RM_CommandFilterArgGet(RedisModuleCommandFilterCtx *filter, int pos)
+const RedisModuleString *RM_CommandFilterArgGet(RedisModuleCommandFilterCtx *fctx, int pos)
 {
-    if (pos < 0 || pos >= filter->argc) return NULL;
-    return filter->argv[pos];
+    if (pos < 0 || pos >= fctx->argc) return NULL;
+    return fctx->argv[pos];
 }
 
 /* Modify the filtered command by inserting a new argument at the specified
@@ -4894,18 +4939,18 @@ const RedisModuleString *RM_CommandFilterArgGet(RedisModuleCommandFilterCtx *fil
  * allocated, freed or used elsewhere.
  */
 
-int RM_CommandFilterArgInsert(RedisModuleCommandFilterCtx *filter, int pos, RedisModuleString *arg)
+int RM_CommandFilterArgInsert(RedisModuleCommandFilterCtx *fctx, int pos, RedisModuleString *arg)
 {
     int i;
 
-    if (pos < 0 || pos > filter->argc) return REDISMODULE_ERR;
+    if (pos < 0 || pos > fctx->argc) return REDISMODULE_ERR;
 
-    filter->argv = zrealloc(filter->argv, (filter->argc+1)*sizeof(RedisModuleString *));
-    for (i = filter->argc; i > pos; i--) {
-        filter->argv[i] = filter->argv[i-1];
+    fctx->argv = zrealloc(fctx->argv, (fctx->argc+1)*sizeof(RedisModuleString *));
+    for (i = fctx->argc; i > pos; i--) {
+        fctx->argv[i] = fctx->argv[i-1];
     }
-    filter->argv[pos] = arg;
-    filter->argc++;
+    fctx->argv[pos] = arg;
+    fctx->argc++;
 
     return REDISMODULE_OK;
 }
@@ -4916,12 +4961,12 @@ int RM_CommandFilterArgInsert(RedisModuleCommandFilterCtx *filter, int pos, Redi
  * or used elsewhere.
  */
 
-int RM_CommandFilterArgReplace(RedisModuleCommandFilterCtx *filter, int pos, RedisModuleString *arg)
+int RM_CommandFilterArgReplace(RedisModuleCommandFilterCtx *fctx, int pos, RedisModuleString *arg)
 {
-    if (pos < 0 || pos >= filter->argc) return REDISMODULE_ERR;
+    if (pos < 0 || pos >= fctx->argc) return REDISMODULE_ERR;
 
-    decrRefCount(filter->argv[pos]);
-    filter->argv[pos] = arg;
+    decrRefCount(fctx->argv[pos]);
+    fctx->argv[pos] = arg;
 
     return REDISMODULE_OK;
 }
@@ -4929,16 +4974,16 @@ int RM_CommandFilterArgReplace(RedisModuleCommandFilterCtx *filter, int pos, Red
 /* Modify the filtered command by deleting an argument at the specified
  * position.
  */
-int RM_CommandFilterArgDelete(RedisModuleCommandFilterCtx *filter, int pos)
+int RM_CommandFilterArgDelete(RedisModuleCommandFilterCtx *fctx, int pos)
 {
     int i;
-    if (pos < 0 || pos >= filter->argc) return REDISMODULE_ERR;
+    if (pos < 0 || pos >= fctx->argc) return REDISMODULE_ERR;
 
-    decrRefCount(filter->argv[pos]);
-    for (i = pos; i < filter->argc-1; i++) {
-        filter->argv[i] = filter->argv[i+1];
+    decrRefCount(fctx->argv[pos]);
+    for (i = pos; i < fctx->argc-1; i++) {
+        fctx->argv[i] = fctx->argv[i+1];
     }
-    filter->argc--;
+    fctx->argc--;
 
     return REDISMODULE_OK;
 }
@@ -5041,6 +5086,7 @@ void moduleLoadFromQueue(void) {
 
 void moduleFreeModuleStructure(struct RedisModule *module) {
     listRelease(module->types);
+    listRelease(module->filters);
     sdsfree(module->name);
     zfree(module);
 }
@@ -5132,6 +5178,7 @@ int moduleUnload(sds name) {
     moduleUnregisterCommands(module);
     moduleUnregisterSharedAPI(module);
     moduleUnregisterUsedAPI(module);
+    moduleUnregisterFilters(module);
 
     /* Remove any notification subscribers this module might have */
     moduleUnsubscribeNotifications(module);
@@ -5396,6 +5443,7 @@ void moduleRegisterCoreAPI(void) {
     REGISTER_API(ExportSharedAPI);
     REGISTER_API(GetSharedAPI);
     REGISTER_API(RegisterCommandFilter);
+    REGISTER_API(UnregisterCommandFilter);
     REGISTER_API(CommandFilterArgsCount);
     REGISTER_API(CommandFilterArgGet);
     REGISTER_API(CommandFilterArgInsert);
diff --git a/src/modules/hellofilter.c b/src/modules/hellofilter.c
index d5dd405aa..9cd440df2 100644
--- a/src/modules/hellofilter.c
+++ b/src/modules/hellofilter.c
@@ -7,10 +7,27 @@ static RedisModuleString *log_key_name;
 
 static const char log_command_name[] = "hellofilter.log";
 static const char ping_command_name[] = "hellofilter.ping";
+static const char unregister_command_name[] = "hellofilter.unregister";
 static int in_module = 0;
 
+static RedisModuleCommandFilter *filter = NULL;
+
+int HelloFilter_UnregisterCommand(RedisModuleCtx *ctx, RedisModuleString **argv, int argc)
+{
+    (void) argc;
+    (void) argv;
+
+    RedisModule_ReplyWithLongLong(ctx,
+            RedisModule_UnregisterCommandFilter(ctx, filter));
+
+    return REDISMODULE_OK;
+}
+
 int HelloFilter_PingCommand(RedisModuleCtx *ctx, RedisModuleString **argv, int argc)
 {
+    (void) argc;
+    (void) argv;
+
     RedisModuleCallReply *reply = RedisModule_Call(ctx, "ping", "c", "@log");
     if (reply) {
         RedisModule_ReplyWithCallReply(ctx, reply);
@@ -115,11 +132,15 @@ int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc)
             return REDISMODULE_ERR;
 
     if (RedisModule_CreateCommand(ctx,ping_command_name,
-                HelloFilter_PingCommand,"write deny-oom",1,1,1) == REDISMODULE_ERR)
+                HelloFilter_PingCommand,"deny-oom",1,1,1) == REDISMODULE_ERR)
             return REDISMODULE_ERR;
 
-    if (RedisModule_RegisterCommandFilter(ctx, HelloFilter_CommandFilter)
-            == REDISMODULE_ERR) return REDISMODULE_ERR;
+    if (RedisModule_CreateCommand(ctx,unregister_command_name,
+                HelloFilter_UnregisterCommand,"write deny-oom",1,1,1) == REDISMODULE_ERR)
+            return REDISMODULE_ERR;
+
+    if ((filter = RedisModule_RegisterCommandFilter(ctx, HelloFilter_CommandFilter))
+            == NULL) return REDISMODULE_ERR;
 
     return REDISMODULE_OK;
 }
diff --git a/src/redismodule.h b/src/redismodule.h
index 5df83ae6a..37b7d0d59 100644
--- a/src/redismodule.h
+++ b/src/redismodule.h
@@ -151,6 +151,7 @@ typedef struct RedisModuleClusterInfo RedisModuleClusterInfo;
 typedef struct RedisModuleDict RedisModuleDict;
 typedef struct RedisModuleDictIter RedisModuleDictIter;
 typedef struct RedisModuleCommandFilterCtx RedisModuleCommandFilterCtx;
+typedef struct RedisModuleCommandFilter RedisModuleCommandFilter;
 
 typedef int (*RedisModuleCmdFunc)(RedisModuleCtx *ctx, RedisModuleString **argv, int argc);
 typedef void (*RedisModuleDisconnectFunc)(RedisModuleCtx *ctx, RedisModuleBlockedClient *bc);
@@ -339,12 +340,13 @@ void REDISMODULE_API_FUNC(RedisModule_SetDisconnectCallback)(RedisModuleBlockedC
 void REDISMODULE_API_FUNC(RedisModule_SetClusterFlags)(RedisModuleCtx *ctx, uint64_t flags);
 int REDISMODULE_API_FUNC(RedisModule_ExportSharedAPI)(RedisModuleCtx *ctx, const char *apiname, void *func);
 void *REDISMODULE_API_FUNC(RedisModule_GetSharedAPI)(RedisModuleCtx *ctx, const char *apiname);
-int REDISMODULE_API_FUNC(RedisModule_RegisterCommandFilter)(RedisModuleCtx *ctx, RedisModuleCommandFilterFunc cb);
-int REDISMODULE_API_FUNC(RedisModule_CommandFilterArgsCount)(RedisModuleCommandFilterCtx *filter);
-const RedisModuleString *REDISMODULE_API_FUNC(RedisModule_CommandFilterArgGet)(RedisModuleCommandFilterCtx *filter, int pos);
-int REDISMODULE_API_FUNC(RedisModule_CommandFilterArgInsert)(RedisModuleCommandFilterCtx *filter, int pos, RedisModuleString *arg);
-int REDISMODULE_API_FUNC(RedisModule_CommandFilterArgReplace)(RedisModuleCommandFilterCtx *filter, int pos, RedisModuleString *arg);
-int REDISMODULE_API_FUNC(RedisModule_CommandFilterArgDelete)(RedisModuleCommandFilterCtx *filter, int pos);
+RedisModuleCommandFilter *REDISMODULE_API_FUNC(RedisModule_RegisterCommandFilter)(RedisModuleCtx *ctx, RedisModuleCommandFilterFunc cb);
+int REDISMODULE_API_FUNC(RedisModule_UnregisterCommandFilter)(RedisModuleCtx *ctx, RedisModuleCommandFilter *filter);
+int REDISMODULE_API_FUNC(RedisModule_CommandFilterArgsCount)(RedisModuleCommandFilterCtx *fctx);
+const RedisModuleString *REDISMODULE_API_FUNC(RedisModule_CommandFilterArgGet)(RedisModuleCommandFilterCtx *fctx, int pos);
+int REDISMODULE_API_FUNC(RedisModule_CommandFilterArgInsert)(RedisModuleCommandFilterCtx *fctx, int pos, RedisModuleString *arg);
+int REDISMODULE_API_FUNC(RedisModule_CommandFilterArgReplace)(RedisModuleCommandFilterCtx *fctx, int pos, RedisModuleString *arg);
+int REDISMODULE_API_FUNC(RedisModule_CommandFilterArgDelete)(RedisModuleCommandFilterCtx *fctx, int pos);
 #endif
 
 /* This is included inline inside each Redis module. */
@@ -508,6 +510,7 @@ static int RedisModule_Init(RedisModuleCtx *ctx, const char *name, int ver, int
     REDISMODULE_GET_API(ExportSharedAPI);
     REDISMODULE_GET_API(GetSharedAPI);
     REDISMODULE_GET_API(RegisterCommandFilter);
+    REDISMODULE_GET_API(UnregisterCommandFilter);
     REDISMODULE_GET_API(CommandFilterArgsCount);
     REDISMODULE_GET_API(CommandFilterArgGet);
     REDISMODULE_GET_API(CommandFilterArgInsert);
diff --git a/tests/modules/commandfilter.tcl b/tests/modules/commandfilter.tcl
index 47d9c302c..8645d8279 100644
--- a/tests/modules/commandfilter.tcl
+++ b/tests/modules/commandfilter.tcl
@@ -42,4 +42,26 @@ start_server {tags {"modules"}} {
         r eval "redis.call('hellofilter.ping')" 0
         r lrange log-key 0 -1
     } "{ping @log}"
+
+    test {Command Filter is unregistered implicitly on module unload} {
+        r del log-key
+        r module unload hellofilter
+        r set mykey @log
+        r lrange log-key 0 -1
+    } {}
+
+    r module load $testmodule log-key-2
+
+    test {Command Filter unregister works as expected} {
+        # Validate reloading succeeded
+        r set mykey @log
+        assert_equal "{set mykey @log}" [r lrange log-key-2 0 -1]
+
+        # Unregister
+        r hellofilter.unregister
+        r del log-key-2
+
+        r set mykey @log
+        r lrange log-key-2 0 -1
+    } {}
 } 

From 4c49e7ad6fbfc7bd75538a30fcdf7ce74bbab050 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Thu, 21 Mar 2019 17:18:24 +0100
Subject: [PATCH 48/60] Mostly aesthetic changes to restartAOF().

See #3829.
---
 src/replication.c | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/src/replication.c b/src/replication.c
index 59e42e561..c25e7fa6f 100644
--- a/src/replication.c
+++ b/src/replication.c
@@ -1093,12 +1093,16 @@ void replicationCreateMasterClient(int fd, int dbid) {
 void restartAOF() {
     unsigned int tries, max_tries = 10;
     for (tries = 0; tries < max_tries; ++tries) {
-        if (tries) sleep(1);
         if (startAppendOnly() == C_OK) break;
-        serverLog(LL_WARNING,"Failed enabling the AOF after successful master synchronization! Trying it again in one second.");
+        serverLog(LL_WARNING,
+            "Failed enabling the AOF after successful master synchronization! "
+            "Trying it again in one second.");
+        sleep(1);
     }
     if (tries == max_tries) {
-        serverLog(LL_WARNING,"FATAL: this replica instance finished the synchronization with its master, but the AOF can't be turned on. Exiting now.");
+        serverLog(LL_WARNING,
+            "FATAL: this replica instance finished the synchronization with "
+            "its master, but the AOF can't be turned on. Exiting now.");
         exit(1);
     }
 }

From 7e191d3ea353c34d8f02b44b5afc14fd8b8c6fd1 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Thu, 21 Mar 2019 17:21:25 +0100
Subject: [PATCH 49/60] More sensible name for function: restartAOFAfterSYNC().

Related to #3829.
---
 src/replication.c | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/src/replication.c b/src/replication.c
index c25e7fa6f..a27c29a3b 100644
--- a/src/replication.c
+++ b/src/replication.c
@@ -1090,7 +1090,11 @@ void replicationCreateMasterClient(int fd, int dbid) {
     if (dbid != -1) selectDb(server.master,dbid);
 }
 
-void restartAOF() {
+/* This function will try to re-enable the AOF file after the
+ * master-replica synchronization: if it fails after multiple attempts
+ * the replica cannot be considered reliable and exists with an
+ * error. */
+void restartAOFAfterSYNC() {
     unsigned int tries, max_tries = 10;
     for (tries = 0; tries < max_tries; ++tries) {
         if (startAppendOnly() == C_OK) break;
@@ -1289,7 +1293,7 @@ void readSyncBulkPayload(aeEventLoop *el, int fd, void *privdata, int mask) {
             cancelReplicationHandshake();
             /* Re-enable the AOF if we disabled it earlier, in order to restore
              * the original configuration. */
-            if (aof_is_enabled) restartAOF();
+            if (aof_is_enabled) restartAOFAfterSYNC();
             return;
         }
         /* Final setup of the connected slave <- master link */
@@ -1314,7 +1318,7 @@ void readSyncBulkPayload(aeEventLoop *el, int fd, void *privdata, int mask) {
         /* Restart the AOF subsystem now that we finished the sync. This
          * will trigger an AOF rewrite, and when done will start appending
          * to the new file. */
-        if (aof_is_enabled) restartAOF();
+        if (aof_is_enabled) restartAOFAfterSYNC();
     }
     return;
 

From b8568a98fd39ecd9e540d74b79ee97a68b0ce8be Mon Sep 17 00:00:00 2001
From: Yossi Gottlieb <yossigo@gmail.com>
Date: Thu, 21 Mar 2019 19:45:41 +0200
Subject: [PATCH 50/60] CommandFilter API: fix UnregisterCommandFilter.

---
 src/module.c | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/src/module.c b/src/module.c
index ad7bba2eb..a54bd1ad8 100644
--- a/src/module.c
+++ b/src/module.c
@@ -4887,9 +4887,8 @@ int RM_UnregisterCommandFilter(RedisModuleCtx *ctx, RedisModuleCommandFilter *fi
     listDelNode(moduleCommandFilters,ln);
     
     ln = listSearchKey(ctx->module->filters,filter);
-    if (ln) {
-        listDelNode(moduleCommandFilters,ln);
-    }
+    if (!ln) return REDISMODULE_ERR;    /* Shouldn't happen */
+    listDelNode(ctx->module->filters,ln);
 
     return REDISMODULE_OK;
 }

From 898677d59e6b77a11aced5d68b32da7e94c075fe Mon Sep 17 00:00:00 2001
From: Yossi Gottlieb <yossigo@gmail.com>
Date: Thu, 21 Mar 2019 19:47:43 +0200
Subject: [PATCH 51/60] CommandFilter API: REDISMODULE_CMDFILTER_NOSELF.

Add a flag to automatically protect filters from being called
recursively by their own module.
---
 src/module.c                    | 28 +++++++++++++++++++++++++---
 src/modules/hellofilter.c       | 15 +++++++++------
 src/redismodule.h               |  7 ++++++-
 tests/modules/commandfilter.tcl | 27 ++++++++++++++++++++++-----
 4 files changed, 62 insertions(+), 15 deletions(-)

diff --git a/src/module.c b/src/module.c
index a54bd1ad8..4ff865d2c 100644
--- a/src/module.c
+++ b/src/module.c
@@ -50,6 +50,7 @@ struct RedisModule {
     list *usedby;   /* List of modules using APIs from this one. */
     list *using;    /* List of modules we use some APIs of. */
     list *filters;  /* List of filters the module has registered. */
+    int in_call;    /* RM_Call() nesting level */
 };
 typedef struct RedisModule RedisModule;
 
@@ -283,6 +284,8 @@ typedef struct RedisModuleCommandFilter {
     RedisModule *module;
     /* Filter callback function */
     RedisModuleCommandFilterFunc callback;
+    /* REDISMODULE_CMDFILTER_* flags */
+    int flags;
 } RedisModuleCommandFilter;
 
 /* Registered filters */
@@ -2756,6 +2759,8 @@ RedisModuleCallReply *RM_Call(RedisModuleCtx *ctx, const char *cmdname, const ch
     c->db = ctx->client->db;
     c->argv = argv;
     c->argc = argc;
+    if (ctx->module) ctx->module->in_call++;
+
     /* We handle the above format error only when the client is setup so that
      * we can free it normally. */
     if (argv == NULL) goto cleanup;
@@ -2822,6 +2827,7 @@ RedisModuleCallReply *RM_Call(RedisModuleCtx *ctx, const char *cmdname, const ch
     autoMemoryAdd(ctx,REDISMODULE_AM_REPLY,reply);
 
 cleanup:
+    if (ctx->module) ctx->module->in_call--;
     freeClient(c);
     return reply;
 }
@@ -4857,17 +4863,27 @@ int moduleUnregisterFilters(RedisModule *module) {
  *
  * Note that in the above use case, if `MODULE.SET` itself uses
  * `RedisModule_Call()` the filter will be applied on that call as well.  If
- * that is not desired, the module itself is responsible for maintaining a flag
- * to identify and avoid this form of re-entrancy.
+ * that is not desired, the `REDISMODULE_CMDFILTER_NOSELF` flag can be set when
+ * registering the filter.
+ *
+ * The `REDISMODULE_CMDFILTER_NOSELF` flag prevents execution flows that
+ * originate from the module's own `RM_Call()` from reaching the filter.  This
+ * flag is effective for all execution flows, including nested ones, as long as
+ * the execution begins from the module's command context or a thread-safe
+ * context that is associated with a blocking command.
+ *
+ * Detached thread-safe contexts are *not* associated with the module and cannot
+ * be protected by this flag.
  *
  * If multiple filters are registered (by the same or different modules), they
  * are executed in the order of registration.
  */
 
-RedisModuleCommandFilter *RM_RegisterCommandFilter(RedisModuleCtx *ctx, RedisModuleCommandFilterFunc callback) {
+RedisModuleCommandFilter *RM_RegisterCommandFilter(RedisModuleCtx *ctx, RedisModuleCommandFilterFunc callback, int flags) {
     RedisModuleCommandFilter *filter = zmalloc(sizeof(*filter));
     filter->module = ctx->module;
     filter->callback = callback;
+    filter->flags = flags;
 
     listAddNodeTail(moduleCommandFilters, filter);
     listAddNodeTail(ctx->module->filters, filter);
@@ -4908,6 +4924,12 @@ void moduleCallCommandFilters(client *c) {
     while((ln = listNext(&li))) {
         RedisModuleCommandFilter *f = ln->value;
 
+        /* Skip filter if REDISMODULE_CMDFILTER_NOSELF is set and module is
+         * currently processing a command.
+         */
+        if ((f->flags & REDISMODULE_CMDFILTER_NOSELF) && f->module->in_call) continue;
+
+        /* Call filter */
         f->callback(&filter);
     }
 
diff --git a/src/modules/hellofilter.c b/src/modules/hellofilter.c
index 9cd440df2..448e12983 100644
--- a/src/modules/hellofilter.c
+++ b/src/modules/hellofilter.c
@@ -8,7 +8,7 @@ static RedisModuleString *log_key_name;
 static const char log_command_name[] = "hellofilter.log";
 static const char ping_command_name[] = "hellofilter.ping";
 static const char unregister_command_name[] = "hellofilter.unregister";
-static int in_module = 0;
+static int in_log_command = 0;
 
 static RedisModuleCommandFilter *filter = NULL;
 
@@ -57,7 +57,7 @@ int HelloFilter_LogCommand(RedisModuleCtx *ctx, RedisModuleString **argv, int ar
     RedisModule_CloseKey(log);
     RedisModule_FreeString(ctx, s);
 
-    in_module = 1;
+    in_log_command = 1;
 
     size_t cmdlen;
     const char *cmdname = RedisModule_StringPtrLen(argv[1], &cmdlen);
@@ -69,14 +69,14 @@ int HelloFilter_LogCommand(RedisModuleCtx *ctx, RedisModuleString **argv, int ar
         RedisModule_ReplyWithSimpleString(ctx, "Unknown command or invalid arguments");
     }
 
-    in_module = 0;
+    in_log_command = 0;
 
     return REDISMODULE_OK;
 }
 
 void HelloFilter_CommandFilter(RedisModuleCommandFilterCtx *filter)
 {
-    if (in_module) return;  /* don't process our own RM_Call() */
+    if (in_log_command) return;  /* don't process our own RM_Call() from HelloFilter_LogCommand() */
 
     /* Fun manipulations:
      * - Remove @delme
@@ -120,12 +120,14 @@ int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc)
     if (RedisModule_Init(ctx,"hellofilter",1,REDISMODULE_APIVER_1)
             == REDISMODULE_ERR) return REDISMODULE_ERR;
 
-    if (argc != 1) {
+    if (argc != 2) {
         RedisModule_Log(ctx, "warning", "Log key name not specified");
         return REDISMODULE_ERR;
     }
 
+    long long noself = 0;
     log_key_name = RedisModule_CreateStringFromString(ctx, argv[0]);
+    RedisModule_StringToLongLong(argv[1], &noself);
 
     if (RedisModule_CreateCommand(ctx,log_command_name,
                 HelloFilter_LogCommand,"write deny-oom",1,1,1) == REDISMODULE_ERR)
@@ -139,7 +141,8 @@ int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc)
                 HelloFilter_UnregisterCommand,"write deny-oom",1,1,1) == REDISMODULE_ERR)
             return REDISMODULE_ERR;
 
-    if ((filter = RedisModule_RegisterCommandFilter(ctx, HelloFilter_CommandFilter))
+    if ((filter = RedisModule_RegisterCommandFilter(ctx, HelloFilter_CommandFilter, 
+                    noself ? REDISMODULE_CMDFILTER_NOSELF : 0))
             == NULL) return REDISMODULE_ERR;
 
     return REDISMODULE_OK;
diff --git a/src/redismodule.h b/src/redismodule.h
index 37b7d0d59..e567743a4 100644
--- a/src/redismodule.h
+++ b/src/redismodule.h
@@ -132,6 +132,11 @@
  * of timers that are going to expire, sorted by expire time. */
 typedef uint64_t RedisModuleTimerID;
 
+/* CommandFilter Flags */
+
+/* Do filter RedisModule_Call() commands initiated by module itself. */
+#define REDISMODULE_CMDFILTER_NOSELF    (1<<0)
+
 /* ------------------------- End of common defines ------------------------ */
 
 #ifndef REDISMODULE_CORE
@@ -340,7 +345,7 @@ void REDISMODULE_API_FUNC(RedisModule_SetDisconnectCallback)(RedisModuleBlockedC
 void REDISMODULE_API_FUNC(RedisModule_SetClusterFlags)(RedisModuleCtx *ctx, uint64_t flags);
 int REDISMODULE_API_FUNC(RedisModule_ExportSharedAPI)(RedisModuleCtx *ctx, const char *apiname, void *func);
 void *REDISMODULE_API_FUNC(RedisModule_GetSharedAPI)(RedisModuleCtx *ctx, const char *apiname);
-RedisModuleCommandFilter *REDISMODULE_API_FUNC(RedisModule_RegisterCommandFilter)(RedisModuleCtx *ctx, RedisModuleCommandFilterFunc cb);
+RedisModuleCommandFilter *REDISMODULE_API_FUNC(RedisModule_RegisterCommandFilter)(RedisModuleCtx *ctx, RedisModuleCommandFilterFunc cb, int flags);
 int REDISMODULE_API_FUNC(RedisModule_UnregisterCommandFilter)(RedisModuleCtx *ctx, RedisModuleCommandFilter *filter);
 int REDISMODULE_API_FUNC(RedisModule_CommandFilterArgsCount)(RedisModuleCommandFilterCtx *fctx);
 const RedisModuleString *REDISMODULE_API_FUNC(RedisModule_CommandFilterArgGet)(RedisModuleCommandFilterCtx *fctx, int pos);
diff --git a/tests/modules/commandfilter.tcl b/tests/modules/commandfilter.tcl
index 8645d8279..1e5c41d2b 100644
--- a/tests/modules/commandfilter.tcl
+++ b/tests/modules/commandfilter.tcl
@@ -1,7 +1,7 @@
 set testmodule [file normalize src/modules/hellofilter.so]
 
 start_server {tags {"modules"}} {
-    r module load $testmodule log-key
+    r module load $testmodule log-key 0
 
     test {Command Filter handles redirected commands} {
         r set mykey @log
@@ -50,18 +50,35 @@ start_server {tags {"modules"}} {
         r lrange log-key 0 -1
     } {}
 
-    r module load $testmodule log-key-2
+    r module load $testmodule log-key 0
 
     test {Command Filter unregister works as expected} {
         # Validate reloading succeeded
+        r del log-key
         r set mykey @log
-        assert_equal "{set mykey @log}" [r lrange log-key-2 0 -1]
+        assert_equal "{set mykey @log}" [r lrange log-key 0 -1]
 
         # Unregister
         r hellofilter.unregister
-        r del log-key-2
+        r del log-key
 
         r set mykey @log
-        r lrange log-key-2 0 -1
+        r lrange log-key 0 -1
     } {}
+
+    r module unload hellofilter
+    r module load $testmodule log-key 1
+
+    test {Command Filter REDISMODULE_CMDFILTER_NOSELF works as expected} {
+        r set mykey @log
+        assert_equal "{set mykey @log}" [r lrange log-key 0 -1]
+
+        r del log-key
+        r hellofilter.ping
+        assert_equal {} [r lrange log-key 0 -1]
+
+        r eval "redis.call('hellofilter.ping')" 0
+        assert_equal {} [r lrange log-key 0 -1]
+    }
+
 } 

From eb40ac6c8e1617962fc1b0d8fcc9f82e3ab0227c Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Tue, 21 Mar 2017 07:20:02 -0700
Subject: [PATCH 52/60] diskless fork kept streaming RDB to a disconnected
 slave

---
 src/networking.c | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/src/networking.c b/src/networking.c
index c08f43e6a..09cbff387 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -911,6 +911,16 @@ void unlinkClient(client *c) {
             c->client_list_node = NULL;
         }
 
+        /* In the case of diskless replication the fork is writing to the
+         * sockets and just closing the fd isn't enough, if we don't also
+         * shutdown the socket the fork will continue to write to the slave
+         * and the salve will only find out that it was disconnected when
+         * it will finish reading the rdb. */
+        if ((c->flags & CLIENT_SLAVE) &&
+            (c->replstate == SLAVE_STATE_WAIT_BGSAVE_END)) {
+            shutdown(c->fd, SHUT_RDWR);
+        }
+
         /* Unregister async I/O handlers and close the socket. */
         aeDeleteFileEvent(server.el,c->fd,AE_READABLE);
         aeDeleteFileEvent(server.el,c->fd,AE_WRITABLE);

From 1a24f23a50095d95beba39536d59d4e777418252 Mon Sep 17 00:00:00 2001
From: Dvir Volk <dvirsky@dvirsky-macbookpro.roam.corp.google.com>
Date: Thu, 21 Mar 2019 20:33:11 +0200
Subject: [PATCH 53/60] Renamed event name from "miss" to "keymiss"

---
 src/db.c                 | 8 ++++----
 src/modules/testmodule.c | 2 +-
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/db.c b/src/db.c
index afe181281..b537a29a4 100644
--- a/src/db.c
+++ b/src/db.c
@@ -83,7 +83,7 @@ robj *lookupKey(redisDb *db, robj *key, int flags) {
  * 1. A key gets expired if it reached it's TTL.
  * 2. The key last access time is updated.
  * 3. The global keys hits/misses stats are updated (reported in INFO).
- * 4. If keyspace notifications are enabled, a "miss" notification is fired.
+ * 4. If keyspace notifications are enabled, a "keymiss" notification is fired.
  *
  * This API should not be used when we write to the key after obtaining
  * the object linked to the key, but only for read only operations.
@@ -107,7 +107,7 @@ robj *lookupKeyReadWithFlags(redisDb *db, robj *key, int flags) {
          * to return NULL ASAP. */
         if (server.masterhost == NULL) {
             server.stat_keyspace_misses++;
-            notifyKeyspaceEvent(NOTIFY_KEY_MISS, "miss", key, db->id);
+            notifyKeyspaceEvent(NOTIFY_KEY_MISS, "keymiss", key, db->id);
             return NULL;
         }
 
@@ -129,14 +129,14 @@ robj *lookupKeyReadWithFlags(redisDb *db, robj *key, int flags) {
             server.current_client->cmd->flags & CMD_READONLY)
         {
             server.stat_keyspace_misses++;
-            notifyKeyspaceEvent(NOTIFY_KEY_MISS, "miss", key, db->id);
+            notifyKeyspaceEvent(NOTIFY_KEY_MISS, "keymiss", key, db->id);
             return NULL;
         }
     }
     val = lookupKey(db,key,flags);
     if (val == NULL) {
         server.stat_keyspace_misses++;
-        notifyKeyspaceEvent(NOTIFY_KEY_MISS, "miss", key, db->id);
+        notifyKeyspaceEvent(NOTIFY_KEY_MISS, "keymiss", key, db->id);
     }
     else
         server.stat_keyspace_hits++;
diff --git a/src/modules/testmodule.c b/src/modules/testmodule.c
index af78d21d7..5381380e5 100644
--- a/src/modules/testmodule.c
+++ b/src/modules/testmodule.c
@@ -188,7 +188,7 @@ int TestNotifications(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
     RedisModule_Call(ctx, "LPUSH", "cc", "l", "y");
     RedisModule_Call(ctx, "LPUSH", "cc", "l", "y");
 
-    /* Miss some keys intentionally so we will get a "miss" notification. */
+    /* Miss some keys intentionally so we will get a "keymiss" notification. */
     RedisModule_Call(ctx, "GET", "c", "nosuchkey");
     RedisModule_Call(ctx, "SMEMBERS", "c", "nosuchkey");
 

From 2d4635b483460c18335089141db306a3bc122123 Mon Sep 17 00:00:00 2001
From: Yossi Gottlieb <yossigo@gmail.com>
Date: Sun, 24 Mar 2019 12:00:33 +0200
Subject: [PATCH 54/60] fix: missing initialization.

---
 src/module.c                                               | 1 +
 src/modules/hellofilter.c => tests/modules/commandfilter.c | 0
 tests/{modules => unit/moduleapi}/commandfilter.tcl        | 0
 3 files changed, 1 insertion(+)
 rename src/modules/hellofilter.c => tests/modules/commandfilter.c (100%)
 rename tests/{modules => unit/moduleapi}/commandfilter.tcl (100%)

diff --git a/src/module.c b/src/module.c
index ff7f27cdd..f468d4996 100644
--- a/src/module.c
+++ b/src/module.c
@@ -753,6 +753,7 @@ void RM_SetModuleAttribs(RedisModuleCtx *ctx, const char *name, int ver, int api
     module->usedby = listCreate();
     module->using = listCreate();
     module->filters = listCreate();
+    module->in_call = 0;
     ctx->module = module;
 }
 
diff --git a/src/modules/hellofilter.c b/tests/modules/commandfilter.c
similarity index 100%
rename from src/modules/hellofilter.c
rename to tests/modules/commandfilter.c
diff --git a/tests/modules/commandfilter.tcl b/tests/unit/moduleapi/commandfilter.tcl
similarity index 100%
rename from tests/modules/commandfilter.tcl
rename to tests/unit/moduleapi/commandfilter.tcl

From a631f66710954a35f4854a98d395e29b898ff6c6 Mon Sep 17 00:00:00 2001
From: Yossi Gottlieb <yossigo@gmail.com>
Date: Sun, 24 Mar 2019 12:03:03 +0200
Subject: [PATCH 55/60] Add runtest-moduleapi with commandfilter coverage.

---
 runtest-moduleapi                      | 16 +++++++++++++++
 src/modules/Makefile                   |  7 +------
 tests/modules/Makefile                 | 24 ++++++++++++++++++++++
 tests/modules/commandfilter.c          | 28 +++++++++++++-------------
 tests/test_helper.tcl                  |  1 -
 tests/unit/moduleapi/commandfilter.tcl | 16 +++++++--------
 6 files changed, 63 insertions(+), 29 deletions(-)
 create mode 100755 runtest-moduleapi
 create mode 100644 tests/modules/Makefile

diff --git a/runtest-moduleapi b/runtest-moduleapi
new file mode 100755
index 000000000..84cdb9bb8
--- /dev/null
+++ b/runtest-moduleapi
@@ -0,0 +1,16 @@
+#!/bin/sh
+TCL_VERSIONS="8.5 8.6"
+TCLSH=""
+
+for VERSION in $TCL_VERSIONS; do
+	TCL=`which tclsh$VERSION 2>/dev/null` && TCLSH=$TCL
+done
+
+if [ -z $TCLSH ]
+then
+    echo "You need tcl 8.5 or newer in order to run the Redis test"
+    exit 1
+fi
+
+make -C tests/modules && \
+$TCLSH tests/test_helper.tcl --single unit/moduleapi/commandfilter "${@}"
diff --git a/src/modules/Makefile b/src/modules/Makefile
index 537aa0daf..4f6b50f2e 100644
--- a/src/modules/Makefile
+++ b/src/modules/Makefile
@@ -13,7 +13,7 @@ endif
 
 .SUFFIXES: .c .so .xo .o
 
-all: helloworld.so hellotype.so helloblock.so testmodule.so hellocluster.so hellotimer.so hellodict.so hellofilter.so
+all: helloworld.so hellotype.so helloblock.so testmodule.so hellocluster.so hellotimer.so hellodict.so
 
 .c.xo:
 	$(CC) -I. $(CFLAGS) $(SHOBJ_CFLAGS) -fPIC -c $< -o $@
@@ -47,11 +47,6 @@ hellodict.xo: ../redismodule.h
 
 hellodict.so: hellodict.xo
 
-hellofilter.xo: ../redismodule.h
-
-hellofilter.so: hellofilter.xo
-	$(LD) -o $@ $< $(SHOBJ_LDFLAGS) $(LIBS) -lc
-
 testmodule.xo: ../redismodule.h
 
 testmodule.so: testmodule.xo
diff --git a/tests/modules/Makefile b/tests/modules/Makefile
new file mode 100644
index 000000000..014d20afa
--- /dev/null
+++ b/tests/modules/Makefile
@@ -0,0 +1,24 @@
+
+# find the OS
+uname_S := $(shell sh -c 'uname -s 2>/dev/null || echo not')
+
+# Compile flags for linux / osx
+ifeq ($(uname_S),Linux)
+	SHOBJ_CFLAGS ?= -W -Wall -fno-common -g -ggdb -std=c99 -O2
+	SHOBJ_LDFLAGS ?= -shared
+else
+	SHOBJ_CFLAGS ?= -W -Wall -dynamic -fno-common -g -ggdb -std=c99 -O2
+	SHOBJ_LDFLAGS ?= -bundle -undefined dynamic_lookup
+endif
+
+.SUFFIXES: .c .so .xo .o
+
+all: commandfilter.so
+
+.c.xo:
+	$(CC) -I../../src $(CFLAGS) $(SHOBJ_CFLAGS) -fPIC -c $< -o $@
+
+commandfilter.xo: ../../src/redismodule.h
+
+commandfilter.so: commandfilter.xo
+	$(LD) -o $@ $< $(SHOBJ_LDFLAGS) $(LIBS) -lc
diff --git a/tests/modules/commandfilter.c b/tests/modules/commandfilter.c
index 448e12983..d25d49c44 100644
--- a/tests/modules/commandfilter.c
+++ b/tests/modules/commandfilter.c
@@ -1,18 +1,18 @@
 #define REDISMODULE_EXPERIMENTAL_API
-#include "../redismodule.h"
+#include "redismodule.h"
 
 #include <string.h>
 
 static RedisModuleString *log_key_name;
 
-static const char log_command_name[] = "hellofilter.log";
-static const char ping_command_name[] = "hellofilter.ping";
-static const char unregister_command_name[] = "hellofilter.unregister";
+static const char log_command_name[] = "commandfilter.log";
+static const char ping_command_name[] = "commandfilter.ping";
+static const char unregister_command_name[] = "commandfilter.unregister";
 static int in_log_command = 0;
 
 static RedisModuleCommandFilter *filter = NULL;
 
-int HelloFilter_UnregisterCommand(RedisModuleCtx *ctx, RedisModuleString **argv, int argc)
+int CommandFilter_UnregisterCommand(RedisModuleCtx *ctx, RedisModuleString **argv, int argc)
 {
     (void) argc;
     (void) argv;
@@ -23,7 +23,7 @@ int HelloFilter_UnregisterCommand(RedisModuleCtx *ctx, RedisModuleString **argv,
     return REDISMODULE_OK;
 }
 
-int HelloFilter_PingCommand(RedisModuleCtx *ctx, RedisModuleString **argv, int argc)
+int CommandFilter_PingCommand(RedisModuleCtx *ctx, RedisModuleString **argv, int argc)
 {
     (void) argc;
     (void) argv;
@@ -39,7 +39,7 @@ int HelloFilter_PingCommand(RedisModuleCtx *ctx, RedisModuleString **argv, int a
     return REDISMODULE_OK;
 }
 
-int HelloFilter_LogCommand(RedisModuleCtx *ctx, RedisModuleString **argv, int argc)
+int CommandFilter_LogCommand(RedisModuleCtx *ctx, RedisModuleString **argv, int argc)
 {
     RedisModuleString *s = RedisModule_CreateString(ctx, "", 0);
 
@@ -74,9 +74,9 @@ int HelloFilter_LogCommand(RedisModuleCtx *ctx, RedisModuleString **argv, int ar
     return REDISMODULE_OK;
 }
 
-void HelloFilter_CommandFilter(RedisModuleCommandFilterCtx *filter)
+void CommandFilter_CommandFilter(RedisModuleCommandFilterCtx *filter)
 {
-    if (in_log_command) return;  /* don't process our own RM_Call() from HelloFilter_LogCommand() */
+    if (in_log_command) return;  /* don't process our own RM_Call() from CommandFilter_LogCommand() */
 
     /* Fun manipulations:
      * - Remove @delme
@@ -117,7 +117,7 @@ void HelloFilter_CommandFilter(RedisModuleCommandFilterCtx *filter)
 }
 
 int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
-    if (RedisModule_Init(ctx,"hellofilter",1,REDISMODULE_APIVER_1)
+    if (RedisModule_Init(ctx,"commandfilter",1,REDISMODULE_APIVER_1)
             == REDISMODULE_ERR) return REDISMODULE_ERR;
 
     if (argc != 2) {
@@ -130,18 +130,18 @@ int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc)
     RedisModule_StringToLongLong(argv[1], &noself);
 
     if (RedisModule_CreateCommand(ctx,log_command_name,
-                HelloFilter_LogCommand,"write deny-oom",1,1,1) == REDISMODULE_ERR)
+                CommandFilter_LogCommand,"write deny-oom",1,1,1) == REDISMODULE_ERR)
             return REDISMODULE_ERR;
 
     if (RedisModule_CreateCommand(ctx,ping_command_name,
-                HelloFilter_PingCommand,"deny-oom",1,1,1) == REDISMODULE_ERR)
+                CommandFilter_PingCommand,"deny-oom",1,1,1) == REDISMODULE_ERR)
             return REDISMODULE_ERR;
 
     if (RedisModule_CreateCommand(ctx,unregister_command_name,
-                HelloFilter_UnregisterCommand,"write deny-oom",1,1,1) == REDISMODULE_ERR)
+                CommandFilter_UnregisterCommand,"write deny-oom",1,1,1) == REDISMODULE_ERR)
             return REDISMODULE_ERR;
 
-    if ((filter = RedisModule_RegisterCommandFilter(ctx, HelloFilter_CommandFilter, 
+    if ((filter = RedisModule_RegisterCommandFilter(ctx, CommandFilter_CommandFilter, 
                     noself ? REDISMODULE_CMDFILTER_NOSELF : 0))
             == NULL) return REDISMODULE_ERR;
 
diff --git a/tests/test_helper.tcl b/tests/test_helper.tcl
index d2f281526..568eacdee 100644
--- a/tests/test_helper.tcl
+++ b/tests/test_helper.tcl
@@ -63,7 +63,6 @@ set ::all_tests {
     unit/lazyfree
     unit/wait
     unit/pendingquerybuf
-    modules/commandfilter
 }
 # Index to the next test to run in the ::all_tests list.
 set ::next_test 0
diff --git a/tests/unit/moduleapi/commandfilter.tcl b/tests/unit/moduleapi/commandfilter.tcl
index 1e5c41d2b..6078f64f2 100644
--- a/tests/unit/moduleapi/commandfilter.tcl
+++ b/tests/unit/moduleapi/commandfilter.tcl
@@ -1,4 +1,4 @@
-set testmodule [file normalize src/modules/hellofilter.so]
+set testmodule [file normalize tests/modules/commandfilter.so]
 
 start_server {tags {"modules"}} {
     r module load $testmodule log-key 0
@@ -27,7 +27,7 @@ start_server {tags {"modules"}} {
 
     test {Command Filter applies on RM_Call() commands} {
         r del log-key
-        r hellofilter.ping
+        r commandfilter.ping
         r lrange log-key 0 -1
     } "{ping @log}"
 
@@ -39,13 +39,13 @@ start_server {tags {"modules"}} {
 
     test {Command Filter applies on Lua redis.call() that calls a module} {
         r del log-key
-        r eval "redis.call('hellofilter.ping')" 0
+        r eval "redis.call('commandfilter.ping')" 0
         r lrange log-key 0 -1
     } "{ping @log}"
 
     test {Command Filter is unregistered implicitly on module unload} {
         r del log-key
-        r module unload hellofilter
+        r module unload commandfilter
         r set mykey @log
         r lrange log-key 0 -1
     } {}
@@ -59,14 +59,14 @@ start_server {tags {"modules"}} {
         assert_equal "{set mykey @log}" [r lrange log-key 0 -1]
 
         # Unregister
-        r hellofilter.unregister
+        r commandfilter.unregister
         r del log-key
 
         r set mykey @log
         r lrange log-key 0 -1
     } {}
 
-    r module unload hellofilter
+    r module unload commandfilter
     r module load $testmodule log-key 1
 
     test {Command Filter REDISMODULE_CMDFILTER_NOSELF works as expected} {
@@ -74,10 +74,10 @@ start_server {tags {"modules"}} {
         assert_equal "{set mykey @log}" [r lrange log-key 0 -1]
 
         r del log-key
-        r hellofilter.ping
+        r commandfilter.ping
         assert_equal {} [r lrange log-key 0 -1]
 
-        r eval "redis.call('hellofilter.ping')" 0
+        r eval "redis.call('commandfilter.ping')" 0
         assert_equal {} [r lrange log-key 0 -1]
     }
 

From 48d14e5aa71b43de0e7747648908b26e34e59ddb Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Sun, 24 Mar 2019 13:10:55 +0200
Subject: [PATCH 56/60] slave corrupts replication stream when module blocked
 client uses large reply (or POSTPONED_ARRAY)

when redis appends the blocked client reply list to the real client, it didn't
bother to check if it is in fact the master client. so a slave executing that
module command will send replies to the master, causing the master to send the
slave error responses, which will mess up the replication offset
(slave will advance it's replication offset, and the master does not)
---
 src/module.c     |  7 +------
 src/networking.c | 13 +++++++++++++
 src/server.h     |  1 +
 3 files changed, 15 insertions(+), 6 deletions(-)

diff --git a/src/module.c b/src/module.c
index ff7f27cdd..0c8197ac7 100644
--- a/src/module.c
+++ b/src/module.c
@@ -3747,12 +3747,7 @@ void moduleHandleBlockedClients(void) {
          * We need to glue such replies to the client output buffer and
          * free the temporary client we just used for the replies. */
         if (c) {
-            if (bc->reply_client->bufpos)
-                addReplyProto(c,bc->reply_client->buf,
-                                bc->reply_client->bufpos);
-            if (listLength(bc->reply_client->reply))
-                listJoin(c->reply,bc->reply_client->reply);
-            c->reply_bytes += bc->reply_client->reply_bytes;
+            AddReplyFromClient(c, bc->reply_client);
         }
         freeClient(bc->reply_client);
 
diff --git a/src/networking.c b/src/networking.c
index 09cbff387..7fdd1984d 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -744,6 +744,19 @@ void addReplySubcommandSyntaxError(client *c) {
     sdsfree(cmd);
 }
 
+/* Append 'src' client output buffers into 'dst' client output buffers. 
+ * This function clears the output buffers of 'src' */
+void AddReplyFromClient(client *dst, client *src) {
+    if (prepareClientToWrite(dst) != C_OK)
+        return;
+    addReplyProto(dst,src->buf, src->bufpos);
+    if (listLength(src->reply))
+        listJoin(dst->reply,src->reply);
+    dst->reply_bytes += src->reply_bytes;
+    src->reply_bytes = 0;
+    src->bufpos = 0;
+}
+
 /* Copy 'src' client output buffers into 'dst' client output buffers.
  * The function takes care of freeing the old output buffers of the
  * destination client. */
diff --git a/src/server.h b/src/server.h
index 95e0355a6..dfd9f7698 100644
--- a/src/server.h
+++ b/src/server.h
@@ -1529,6 +1529,7 @@ void addReplyNullArray(client *c);
 void addReplyBool(client *c, int b);
 void addReplyVerbatim(client *c, const char *s, size_t len, const char *ext);
 void addReplyProto(client *c, const char *s, size_t len);
+void AddReplyFromClient(client *c, client *src);
 void addReplyBulk(client *c, robj *obj);
 void addReplyBulkCString(client *c, const char *s);
 void addReplyBulkCBuffer(client *c, const void *p, size_t len);

From 41fc29512c191363058175b9b255d54e0918b60f Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Mon, 8 Apr 2019 17:39:22 +0200
Subject: [PATCH 57/60] Fix assert comparison in
 fetchClusterSlotsConfiguration().

---
 src/redis-benchmark.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/redis-benchmark.c b/src/redis-benchmark.c
index 12e9f7e41..4e2662f21 100644
--- a/src/redis-benchmark.c
+++ b/src/redis-benchmark.c
@@ -1192,7 +1192,7 @@ static int fetchClusterSlotsConfiguration(client c) {
     assert(reply->type == REDIS_REPLY_ARRAY);
     for (i = 0; i < reply->elements; i++) {
         redisReply *r = reply->element[i];
-        assert(r->type = REDIS_REPLY_ARRAY);
+        assert(r->type == REDIS_REPLY_ARRAY);
         assert(r->elements >= 3);
         int from, to, slot;
         from = r->element[0]->integer;

From b364f3fc21ca0130f56eb1ec13ff608062e67b96 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Mon, 8 Apr 2019 18:06:50 +0200
Subject: [PATCH 58/60] ACL: regression test for #5998.

---
 tests/unit/acl.tcl | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/tests/unit/acl.tcl b/tests/unit/acl.tcl
index 82c75f82d..90f2c9bbf 100644
--- a/tests/unit/acl.tcl
+++ b/tests/unit/acl.tcl
@@ -108,4 +108,11 @@ start_server {tags {"acl"}} {
         assert_match {*+debug|segfault*} $cmdstr
         assert_match {*+acl*} $cmdstr
     }
+
+    test {ACL regression: memory leaks adding / removing subcommands} {
+        r AUTH default ""
+        r ACL setuser newuser reset -debug +debug|a +debug|b +debug|c
+        r ACL setuser newuser -debug
+        # The test framework will detect a leak if any.
+    }
 }

From 1a505a3ba9b44c0cb420b37458a4ee2f1c4f92b4 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Mon, 8 Apr 2019 18:08:37 +0200
Subject: [PATCH 59/60] ACL: Fix memory leak in
 ACLResetSubcommandsForCommand().

This commit fixes bug reported at #5998. Thanks to @tomcat1102.
---
 src/acl.c          | 2 ++
 tests/unit/acl.tcl | 2 +-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/acl.c b/src/acl.c
index d9f431f4f..0205e51ad 100644
--- a/src/acl.c
+++ b/src/acl.c
@@ -542,6 +542,8 @@ struct redisCommand *ACLLookupCommand(const char *name) {
  * and command ID. */
 void ACLResetSubcommandsForCommand(user *u, unsigned long id) {
     if (u->allowed_subcommands && u->allowed_subcommands[id]) {
+        for (int i = 0; u->allowed_subcommands[id][i]; i++)
+            sdsfree(u->allowed_subcommands[id][i]);
         zfree(u->allowed_subcommands[id]);
         u->allowed_subcommands[id] = NULL;
     }
diff --git a/tests/unit/acl.tcl b/tests/unit/acl.tcl
index 90f2c9bbf..058441433 100644
--- a/tests/unit/acl.tcl
+++ b/tests/unit/acl.tcl
@@ -109,7 +109,7 @@ start_server {tags {"acl"}} {
         assert_match {*+acl*} $cmdstr
     }
 
-    test {ACL regression: memory leaks adding / removing subcommands} {
+    test {ACL #5998 regression: memory leaks adding / removing subcommands} {
         r AUTH default ""
         r ACL setuser newuser reset -debug +debug|a +debug|b +debug|c
         r ACL setuser newuser -debug

From f4ae084e94caab308595824c6ee0857fc1cc3d29 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Wed, 10 Apr 2019 18:53:27 +0200
Subject: [PATCH 60/60] Aesthetic change to #5962 to conform to Redis style.

---
 src/module.c | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/src/module.c b/src/module.c
index 0c8197ac7..1e7c0eca3 100644
--- a/src/module.c
+++ b/src/module.c
@@ -3746,9 +3746,7 @@ void moduleHandleBlockedClients(void) {
          * replies to send to the client in a thread safe context.
          * We need to glue such replies to the client output buffer and
          * free the temporary client we just used for the replies. */
-        if (c) {
-            AddReplyFromClient(c, bc->reply_client);
-        }
+        if (c) AddReplyFromClient(c, bc->reply_client);
         freeClient(bc->reply_client);
 
         if (c != NULL) {