From aa123fff5a49f8d8c25b159c6b53b86b48ae7da5 Mon Sep 17 00:00:00 2001 From: Guy Benoish Date: Wed, 7 Mar 2018 10:40:37 +0700 Subject: [PATCH 01/17] Fix zlexrangespec mem-leak in genericZrangebylexCommand --- src/t_zset.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/t_zset.c b/src/t_zset.c index f7f4c6eb2..fa7793b15 100644 --- a/src/t_zset.c +++ b/src/t_zset.c @@ -2856,7 +2856,10 @@ void genericZrangebylexCommand(client *c, int reverse) { while (remaining) { if (remaining >= 3 && !strcasecmp(c->argv[pos]->ptr,"limit")) { if ((getLongFromObjectOrReply(c, c->argv[pos+1], &offset, NULL) != C_OK) || - (getLongFromObjectOrReply(c, c->argv[pos+2], &limit, NULL) != C_OK)) return; + (getLongFromObjectOrReply(c, c->argv[pos+2], &limit, NULL) != C_OK)) { + zslFreeLexRange(&range); + return; + } pos += 3; remaining -= 3; } else { zslFreeLexRange(&range); From 731fafa3d8f9c75eb0238a98361f8af6425983a5 Mon Sep 17 00:00:00 2001 From: Guy Benoish Date: Wed, 18 Apr 2018 13:01:53 +0300 Subject: [PATCH 02/17] Use memtoll() in 'CONFIG SET client-output-buffer-limit' --- src/config.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/config.c b/src/config.c index a1122d059..a33981c63 100644 --- a/src/config.c +++ b/src/config.c @@ -983,8 +983,8 @@ void configSetCommand(client *c) { int soft_seconds; class = getClientTypeByName(v[j]); - hard = strtoll(v[j+1],NULL,10); - soft = strtoll(v[j+2],NULL,10); + hard = memtoll(v[j+1],NULL); + soft = memtoll(v[j+2],NULL); soft_seconds = strtoll(v[j+3],NULL,10); server.client_obuf_limits[class].hard_limit_bytes = hard; From 8e9c96adc2ebb996e181e065d6ab717d385bdca9 Mon Sep 17 00:00:00 2001 From: Guy Benoish Date: Thu, 13 Dec 2018 13:57:38 +0100 Subject: [PATCH 03/17] Check server.verbosity in RM_LogRaw --- src/module.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/module.c b/src/module.c index 20d159d33..e553bc0d8 100644 --- a/src/module.c +++ b/src/module.c @@ -3421,6 +3421,8 @@ void RM_LogRaw(RedisModule *module, const char *levelstr, const char *fmt, va_li else if (!strcasecmp(levelstr,"warning")) level = LL_WARNING; else level = LL_VERBOSE; /* Default. */ + if (level < server.verbosity) return; + name_len = snprintf(msg, sizeof(msg),"<%s> ", module->name); vsnprintf(msg + name_len, sizeof(msg) - name_len, fmt, ap); serverLogRaw(level,msg); From 2a907e7d0066f30b3dd04d3b6625799be1e1de90 Mon Sep 17 00:00:00 2001 From: MeirShpilraien Date: Sun, 9 Dec 2018 14:32:55 +0200 Subject: [PATCH 04/17] added module ability to register api to be used by other modules --- src/module.c | 115 ++++++++++++++++++++++++++++++++++++++++++++++ src/redismodule.h | 4 ++ src/server.h | 1 + 3 files changed, 120 insertions(+) diff --git a/src/module.c b/src/module.c index 20d159d33..d177af24a 100644 --- a/src/module.c +++ b/src/module.c @@ -47,7 +47,16 @@ struct RedisModule { int ver; /* Module version. We use just progressive integers. */ int apiver; /* Module API version as requested during initialization.*/ list *types; /* Module data types. */ + list *usedBy; /* list of modules names using this module api. */ + list *using; /* list of modules names that this module is using thier api . */ + list *exportedFunctions; /* list of function names exported by this module. */ }; + +struct ModuleExportedApi { + void* funcPointer; + struct RedisModule* module; +}; + typedef struct RedisModule RedisModule; static dict *modules; /* Hash table of modules. SDS -> RedisModule ptr.*/ @@ -700,6 +709,9 @@ void RM_SetModuleAttribs(RedisModuleCtx *ctx, const char *name, int ver, int api module->ver = ver; module->apiver = apiver; module->types = listCreate(); + module->usedBy = listCreate(); + module->using = listCreate(); + module->exportedFunctions = listCreate(); ctx->module = module; } @@ -4615,6 +4627,59 @@ void RM_GetRandomHexChars(char *dst, size_t len) { getRandomHexChars(dst,len); } +/* Used to register an api to be used by other modules. */ +int RM_RegisterApi(RedisModuleCtx *ctx, const char *funcname, void *funcptr) { + struct ModuleExportedApi* eapi = zmalloc(sizeof(*eapi)); + eapi->module = ctx->module; + eapi->funcPointer = funcptr; + if(dictAdd(server.exportedapi, (char*)funcname, eapi) != DICT_OK){ + zfree(eapi); + return REDISMODULE_ERR; + } + listAddNodeHead(ctx->module->exportedFunctions, (char*)funcname); + return REDISMODULE_OK; +} + +static inline int IsModuleInList(list *l, const char* moduleName){ + listIter *iter = listGetIterator(l, AL_START_HEAD); + listNode *node = NULL; + while((node = listNext(iter))){ + char* name = listNodeValue(node); + if(strcmp(name, moduleName) == 0){ + listReleaseIterator(iter); + return 1; + } + } + listReleaseIterator(iter); + return 0; +} + +static inline void RemoveFromList(list *l, const char* moduleName){ + listIter *iter = listGetIterator(l, AL_START_HEAD); + listNode *node = NULL; + while((node = listNext(iter))){ + char* name = listNodeValue(node); + if(strcmp(name, moduleName) == 0){ + listDelNode(l, node); + return; + } + } + listReleaseIterator(iter); +} + +void* RM_GetExportedApi(RedisModuleCtx *ctx, const char *funcname) { + dictEntry* entry = dictFind(server.exportedapi, funcname); + if(!entry){ + return NULL; + } + struct ModuleExportedApi* eapi = dictGetVal(entry); + if(!IsModuleInList(eapi->module->usedBy, ctx->module->name)){ + listAddNodeHead(eapi->module->usedBy, ctx->module->name); + listAddNodeHead(ctx->module->using, eapi->module->name); + } + return eapi->funcPointer; +} + /* -------------------------------------------------------------------------- * Modules API internals * -------------------------------------------------------------------------- */ @@ -4735,6 +4800,28 @@ void moduleUnregisterCommands(struct RedisModule *module) { dictReleaseIterator(di); } +void moduleUnregisterApi(struct RedisModule *module) { + listIter *iter = listGetIterator(module->exportedFunctions, AL_START_HEAD); + listNode* node = NULL; + while((node = listNext(iter))){ + char* functionName = listNodeValue(node); + struct ModuleExportedApi* eapi = dictFetchValue(server.exportedapi, functionName); + serverAssert(eapi); + zfree(eapi); + dictDelete(server.exportedapi, functionName); + } + listReleaseIterator(iter); + iter = listGetIterator(module->using, AL_START_HEAD); + node = NULL; + while((node = listNext(iter))){ + char* moduleName = listNodeValue(node); + struct RedisModule* usingModule = dictFetchValue(modules, moduleName); + serverAssert(usingModule); + RemoveFromList(usingModule->usedBy, module->name); + } + listReleaseIterator(iter); +} + /* Load a module and initialize it. On success C_OK is returned, otherwise * C_ERR is returned. */ int moduleLoad(const char *path, void **module_argv, int module_argc) { @@ -4794,6 +4881,12 @@ int moduleUnload(sds name) { return REDISMODULE_ERR; } + if (listLength(module->usedBy)) { + errno = EPERM; + return REDISMODULE_ERR; + } + + moduleUnregisterApi(module); moduleUnregisterCommands(module); /* Remove any notification subscribers this module might have */ @@ -4826,6 +4919,7 @@ void moduleCommand(client *c) { if (c->argc == 2 && !strcasecmp(subcmd,"help")) { const char *help[] = { "LIST -- Return a list of loaded modules.", +"LISTAPI -- Return a list of exported api.", "LOAD [arg ...] -- Load a module library from .", "UNLOAD -- Unload a module.", NULL @@ -4858,6 +4952,9 @@ NULL case EBUSY: errmsg = "the module exports one or more module-side data types, can't unload"; break; + case EPERM: + errmsg = "the module api is used by other modules, please unload them first and try again."; + break; default: errmsg = "operation not possible."; break; @@ -4879,6 +4976,21 @@ NULL addReplyLongLong(c,module->ver); } dictReleaseIterator(di); + } else if (!strcasecmp(subcmd,"listapi") && c->argc == 3) { + char *moduleName = c->argv[2]->ptr; + struct RedisModule* module = dictFetchValue(modules, moduleName); + if(!module){ + addReplyErrorFormat(c,"Error listing module api: no such module %s", moduleName); + return; + } + addReplyMultiBulkLen(c,listLength(module->exportedFunctions)); + listIter *iter = listGetIterator(module->exportedFunctions, AL_START_HEAD); + listNode* node = NULL; + while((node = listNext(iter))){ + char* functionName = listNodeValue(node); + addReplyBulkCString(c,functionName); + } + listReleaseIterator(iter); } else { addReplySubcommandSyntaxError(c); return; @@ -4894,6 +5006,7 @@ size_t moduleCount(void) { * file so that's easy to seek it to add new entries. */ void moduleRegisterCoreAPI(void) { server.moduleapi = dictCreate(&moduleAPIDictType,NULL); + server.exportedapi = dictCreate(&moduleAPIDictType,NULL); REGISTER_API(Alloc); REGISTER_API(Calloc); REGISTER_API(Realloc); @@ -5044,4 +5157,6 @@ void moduleRegisterCoreAPI(void) { REGISTER_API(DictPrev); REGISTER_API(DictCompareC); REGISTER_API(DictCompare); + REGISTER_API(RegisterApi); + REGISTER_API(GetExportedApi); } diff --git a/src/redismodule.h b/src/redismodule.h index d18c38881..3c76fa02b 100644 --- a/src/redismodule.h +++ b/src/redismodule.h @@ -332,6 +332,8 @@ void REDISMODULE_API_FUNC(RedisModule_GetRandomBytes)(unsigned char *dst, size_t void REDISMODULE_API_FUNC(RedisModule_GetRandomHexChars)(char *dst, size_t len); void REDISMODULE_API_FUNC(RedisModule_SetDisconnectCallback)(RedisModuleBlockedClient *bc, RedisModuleDisconnectFunc callback); void REDISMODULE_API_FUNC(RedisModule_SetClusterFlags)(RedisModuleCtx *ctx, uint64_t flags); +int REDISMODULE_API_FUNC(RedisModule_RegisterApi)(RedisModuleCtx *ctx, const char *funcname, void *funcptr); +void* REDISMODULE_API_FUNC(RedisModule_GetExportedApi)(RedisModuleCtx *ctx, const char *funcname); #endif /* This is included inline inside each Redis module. */ @@ -492,6 +494,8 @@ static int RedisModule_Init(RedisModuleCtx *ctx, const char *name, int ver, int REDISMODULE_GET_API(GetRandomBytes); REDISMODULE_GET_API(GetRandomHexChars); REDISMODULE_GET_API(SetClusterFlags); + REDISMODULE_GET_API(RegisterApi); + REDISMODULE_GET_API(GetExportedApi); #endif if (RedisModule_IsModuleNameBusy && RedisModule_IsModuleNameBusy(name)) return REDISMODULE_ERR; diff --git a/src/server.h b/src/server.h index da4c6d45a..379cda058 100644 --- a/src/server.h +++ b/src/server.h @@ -955,6 +955,7 @@ struct redisServer { int always_show_logo; /* Show logo even for non-stdout logging. */ /* Modules */ dict *moduleapi; /* Exported APIs dictionary for modules. */ + dict *exportedapi; /* Api exported by other modules. */ list *loadmodule_queue; /* List of modules to load at startup. */ int module_blocked_pipe[2]; /* Pipe used to awake the event loop if a client blocked on a module command needs From ea2d2532381bc7ac7a20fb3af16b6f04b46b57b7 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 20 Dec 2018 17:56:38 +0100 Subject: [PATCH 05/17] Revert shared APIs to modify the design. --- src/module.c | 115 ---------------------------------------------- src/redismodule.h | 4 -- src/server.h | 1 - 3 files changed, 120 deletions(-) diff --git a/src/module.c b/src/module.c index d177af24a..20d159d33 100644 --- a/src/module.c +++ b/src/module.c @@ -47,16 +47,7 @@ struct RedisModule { int ver; /* Module version. We use just progressive integers. */ int apiver; /* Module API version as requested during initialization.*/ list *types; /* Module data types. */ - list *usedBy; /* list of modules names using this module api. */ - list *using; /* list of modules names that this module is using thier api . */ - list *exportedFunctions; /* list of function names exported by this module. */ }; - -struct ModuleExportedApi { - void* funcPointer; - struct RedisModule* module; -}; - typedef struct RedisModule RedisModule; static dict *modules; /* Hash table of modules. SDS -> RedisModule ptr.*/ @@ -709,9 +700,6 @@ void RM_SetModuleAttribs(RedisModuleCtx *ctx, const char *name, int ver, int api module->ver = ver; module->apiver = apiver; module->types = listCreate(); - module->usedBy = listCreate(); - module->using = listCreate(); - module->exportedFunctions = listCreate(); ctx->module = module; } @@ -4627,59 +4615,6 @@ void RM_GetRandomHexChars(char *dst, size_t len) { getRandomHexChars(dst,len); } -/* Used to register an api to be used by other modules. */ -int RM_RegisterApi(RedisModuleCtx *ctx, const char *funcname, void *funcptr) { - struct ModuleExportedApi* eapi = zmalloc(sizeof(*eapi)); - eapi->module = ctx->module; - eapi->funcPointer = funcptr; - if(dictAdd(server.exportedapi, (char*)funcname, eapi) != DICT_OK){ - zfree(eapi); - return REDISMODULE_ERR; - } - listAddNodeHead(ctx->module->exportedFunctions, (char*)funcname); - return REDISMODULE_OK; -} - -static inline int IsModuleInList(list *l, const char* moduleName){ - listIter *iter = listGetIterator(l, AL_START_HEAD); - listNode *node = NULL; - while((node = listNext(iter))){ - char* name = listNodeValue(node); - if(strcmp(name, moduleName) == 0){ - listReleaseIterator(iter); - return 1; - } - } - listReleaseIterator(iter); - return 0; -} - -static inline void RemoveFromList(list *l, const char* moduleName){ - listIter *iter = listGetIterator(l, AL_START_HEAD); - listNode *node = NULL; - while((node = listNext(iter))){ - char* name = listNodeValue(node); - if(strcmp(name, moduleName) == 0){ - listDelNode(l, node); - return; - } - } - listReleaseIterator(iter); -} - -void* RM_GetExportedApi(RedisModuleCtx *ctx, const char *funcname) { - dictEntry* entry = dictFind(server.exportedapi, funcname); - if(!entry){ - return NULL; - } - struct ModuleExportedApi* eapi = dictGetVal(entry); - if(!IsModuleInList(eapi->module->usedBy, ctx->module->name)){ - listAddNodeHead(eapi->module->usedBy, ctx->module->name); - listAddNodeHead(ctx->module->using, eapi->module->name); - } - return eapi->funcPointer; -} - /* -------------------------------------------------------------------------- * Modules API internals * -------------------------------------------------------------------------- */ @@ -4800,28 +4735,6 @@ void moduleUnregisterCommands(struct RedisModule *module) { dictReleaseIterator(di); } -void moduleUnregisterApi(struct RedisModule *module) { - listIter *iter = listGetIterator(module->exportedFunctions, AL_START_HEAD); - listNode* node = NULL; - while((node = listNext(iter))){ - char* functionName = listNodeValue(node); - struct ModuleExportedApi* eapi = dictFetchValue(server.exportedapi, functionName); - serverAssert(eapi); - zfree(eapi); - dictDelete(server.exportedapi, functionName); - } - listReleaseIterator(iter); - iter = listGetIterator(module->using, AL_START_HEAD); - node = NULL; - while((node = listNext(iter))){ - char* moduleName = listNodeValue(node); - struct RedisModule* usingModule = dictFetchValue(modules, moduleName); - serverAssert(usingModule); - RemoveFromList(usingModule->usedBy, module->name); - } - listReleaseIterator(iter); -} - /* Load a module and initialize it. On success C_OK is returned, otherwise * C_ERR is returned. */ int moduleLoad(const char *path, void **module_argv, int module_argc) { @@ -4881,12 +4794,6 @@ int moduleUnload(sds name) { return REDISMODULE_ERR; } - if (listLength(module->usedBy)) { - errno = EPERM; - return REDISMODULE_ERR; - } - - moduleUnregisterApi(module); moduleUnregisterCommands(module); /* Remove any notification subscribers this module might have */ @@ -4919,7 +4826,6 @@ void moduleCommand(client *c) { if (c->argc == 2 && !strcasecmp(subcmd,"help")) { const char *help[] = { "LIST -- Return a list of loaded modules.", -"LISTAPI -- Return a list of exported api.", "LOAD [arg ...] -- Load a module library from .", "UNLOAD -- Unload a module.", NULL @@ -4952,9 +4858,6 @@ NULL case EBUSY: errmsg = "the module exports one or more module-side data types, can't unload"; break; - case EPERM: - errmsg = "the module api is used by other modules, please unload them first and try again."; - break; default: errmsg = "operation not possible."; break; @@ -4976,21 +4879,6 @@ NULL addReplyLongLong(c,module->ver); } dictReleaseIterator(di); - } else if (!strcasecmp(subcmd,"listapi") && c->argc == 3) { - char *moduleName = c->argv[2]->ptr; - struct RedisModule* module = dictFetchValue(modules, moduleName); - if(!module){ - addReplyErrorFormat(c,"Error listing module api: no such module %s", moduleName); - return; - } - addReplyMultiBulkLen(c,listLength(module->exportedFunctions)); - listIter *iter = listGetIterator(module->exportedFunctions, AL_START_HEAD); - listNode* node = NULL; - while((node = listNext(iter))){ - char* functionName = listNodeValue(node); - addReplyBulkCString(c,functionName); - } - listReleaseIterator(iter); } else { addReplySubcommandSyntaxError(c); return; @@ -5006,7 +4894,6 @@ size_t moduleCount(void) { * file so that's easy to seek it to add new entries. */ void moduleRegisterCoreAPI(void) { server.moduleapi = dictCreate(&moduleAPIDictType,NULL); - server.exportedapi = dictCreate(&moduleAPIDictType,NULL); REGISTER_API(Alloc); REGISTER_API(Calloc); REGISTER_API(Realloc); @@ -5157,6 +5044,4 @@ void moduleRegisterCoreAPI(void) { REGISTER_API(DictPrev); REGISTER_API(DictCompareC); REGISTER_API(DictCompare); - REGISTER_API(RegisterApi); - REGISTER_API(GetExportedApi); } diff --git a/src/redismodule.h b/src/redismodule.h index 3c76fa02b..d18c38881 100644 --- a/src/redismodule.h +++ b/src/redismodule.h @@ -332,8 +332,6 @@ void REDISMODULE_API_FUNC(RedisModule_GetRandomBytes)(unsigned char *dst, size_t void REDISMODULE_API_FUNC(RedisModule_GetRandomHexChars)(char *dst, size_t len); void REDISMODULE_API_FUNC(RedisModule_SetDisconnectCallback)(RedisModuleBlockedClient *bc, RedisModuleDisconnectFunc callback); void REDISMODULE_API_FUNC(RedisModule_SetClusterFlags)(RedisModuleCtx *ctx, uint64_t flags); -int REDISMODULE_API_FUNC(RedisModule_RegisterApi)(RedisModuleCtx *ctx, const char *funcname, void *funcptr); -void* REDISMODULE_API_FUNC(RedisModule_GetExportedApi)(RedisModuleCtx *ctx, const char *funcname); #endif /* This is included inline inside each Redis module. */ @@ -494,8 +492,6 @@ static int RedisModule_Init(RedisModuleCtx *ctx, const char *name, int ver, int REDISMODULE_GET_API(GetRandomBytes); REDISMODULE_GET_API(GetRandomHexChars); REDISMODULE_GET_API(SetClusterFlags); - REDISMODULE_GET_API(RegisterApi); - REDISMODULE_GET_API(GetExportedApi); #endif if (RedisModule_IsModuleNameBusy && RedisModule_IsModuleNameBusy(name)) return REDISMODULE_ERR; diff --git a/src/server.h b/src/server.h index 379cda058..da4c6d45a 100644 --- a/src/server.h +++ b/src/server.h @@ -955,7 +955,6 @@ struct redisServer { int always_show_logo; /* Show logo even for non-stdout logging. */ /* Modules */ dict *moduleapi; /* Exported APIs dictionary for modules. */ - dict *exportedapi; /* Api exported by other modules. */ list *loadmodule_queue; /* List of modules to load at startup. */ int module_blocked_pipe[2]; /* Pipe used to awake the event loop if a client blocked on a module command needs From f7e99b07f36c8989ca704295318abd50fde4f690 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 20 Dec 2018 12:06:24 +0100 Subject: [PATCH 06/17] Modules shared API: initial core functions. Based on ideas and code in PR #5560 by @MeirShpilraien. --- src/module.c | 86 ++++++++++++++++++++++++++++++++++++++++++++++++++++ src/server.h | 4 ++- 2 files changed, 89 insertions(+), 1 deletion(-) diff --git a/src/module.c b/src/module.c index 20d159d33..2914b5903 100644 --- a/src/module.c +++ b/src/module.c @@ -47,9 +47,21 @@ struct RedisModule { int ver; /* Module version. We use just progressive integers. */ int apiver; /* Module API version as requested during initialization.*/ list *types; /* Module data types. */ + list *usedby; /* List of modules using APIs from this one. */ + list *using; /* List of modules we use some APIs of. */ }; typedef struct RedisModule RedisModule; +/* This represents a shared API. Shared APIs will be used to populate + * the server.sharedapi dictionary, mapping names of APIs exported by + * modules for other modules to use, to their structure specifying the + * function pointer that can be called. */ +struct RedisModuleSharedAPI { + void *func; + RedisModule *module; +}; +typedef struct RedisModuleSharedAPI RedisModuleSharedAPI; + static dict *modules; /* Hash table of modules. SDS -> RedisModule ptr.*/ /* Entries in the context->amqueue array, representing objects to free @@ -700,6 +712,8 @@ void RM_SetModuleAttribs(RedisModuleCtx *ctx, const char *name, int ver, int api module->ver = ver; module->apiver = apiver; module->types = listCreate(); + module->usedby = listCreate(); + module->using = listCreate(); ctx->module = module; } @@ -4615,6 +4629,77 @@ void RM_GetRandomHexChars(char *dst, size_t len) { getRandomHexChars(dst,len); } +/* -------------------------------------------------------------------------- + * Modules API exporting / importing + * -------------------------------------------------------------------------- */ + +/* This function is called by a module in order to export some API with a + * given name. Other modules will be able to use this API by calling the + * symmetrical function RM_GetSharedAPI() and casting the return value to + * the right function pointer. + * + * The function will return REDISMODULE_OK if the name is not already taken, + * otherwise REDISMODULE_ERR will be returned and no operation will be + * performed. + * + * IMPORTANT: the apiname argument should be a string literal with static + * lifetime. The API relies on the fact that it will always be valid in + * the future. */ +int RM_ExportSharedAPI(RedisModuleCtx *ctx, const char *apiname, void *func) { + RedisModuleSharedAPI *sapi = zmalloc(sizeof(*sapi)); + sapi->module = ctx->module; + sapi->func = func; + if (dictAdd(server.sharedapi, (char*)apiname, sapi) != DICT_OK) { + zfree(sapi); + return REDISMODULE_ERR; + } + return REDISMODULE_OK; +} + +/* Request an exported API pointer. The return value is just a void pointer + * that the caller of this function will be required to cast to the right + * function pointer, so this is a private contract between modules. + * + * If the requested API is not available then NULL is returned. Because + * modules can be loaded at different times with different order, this + * function calls should be put inside some module generic API registering + * step, that is called every time a module attempts to execute a + * command that requires external APIs: if some API cannot be resolved, the + * command should return an error. + * + * Here is an exmaple: + * + * int ... myCommandImplementation() { + * if (getExternalAPIs() == 0) { + * reply with an error here if we cannot have the APIs + * } + * // Use the API: + * myFunctionPointer(foo); + * } + * + * And the function registerAPI() is: + * + * int getExternalAPIs(void) { + * static int api_loaded = 0; + * if (api_loaded != 0) return 1; // APIs already resolved. + * + * myFunctionPointer = RedisModule_GetOtherModuleAPI("..."); + * if (myFunctionPointer == NULL) return 0; + * + * return 1; + * } + */ +void *RM_GetSharedAPI(RedisModuleCtx *ctx, const char *apiname) { + dictEntry *de = dictFind(server.sharedapi, apiname); + if (de == NULL) return NULL; + RedisModuleSharedAPI *sapi = dictGetVal(de); + if (listSearchKey(sapi->module->usedby,ctx->module) == NULL) { + listAddNodeTail(sapi->module->usedby,ctx->module); + listAddNodeTail(ctx->module->using,sapi->module); + } + return sapi->func; +} + /* -------------------------------------------------------------------------- * Modules API internals * -------------------------------------------------------------------------- */ @@ -4894,6 +4979,7 @@ size_t moduleCount(void) { * file so that's easy to seek it to add new entries. */ void moduleRegisterCoreAPI(void) { server.moduleapi = dictCreate(&moduleAPIDictType,NULL); + server.sharedapi = dictCreate(&moduleAPIDictType,NULL); REGISTER_API(Alloc); REGISTER_API(Calloc); REGISTER_API(Realloc); diff --git a/src/server.h b/src/server.h index da4c6d45a..3c2ecdd23 100644 --- a/src/server.h +++ b/src/server.h @@ -954,7 +954,9 @@ struct redisServer { size_t initial_memory_usage; /* Bytes used after initialization. */ int always_show_logo; /* Show logo even for non-stdout logging. */ /* Modules */ - dict *moduleapi; /* Exported APIs dictionary for modules. */ + dict *moduleapi; /* Exported core APIs dictionary for modules. */ + dict *sharedapi; /* Like moduleapi but containing the APIs that + modules share with each other. */ list *loadmodule_queue; /* List of modules to load at startup. */ int module_blocked_pipe[2]; /* Pipe used to awake the event loop if a client blocked on a module command needs From a41459ea5b6d8f13da8713ccf49746130f0fd2c8 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 20 Dec 2018 17:16:39 +0100 Subject: [PATCH 07/17] Modules shared API: unregister APIs function. --- src/module.c | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/src/module.c b/src/module.c index 2914b5903..eafdd81f6 100644 --- a/src/module.c +++ b/src/module.c @@ -4700,6 +4700,29 @@ void *RM_GetSharedAPI(RedisModuleCtx *ctx, const char *apiname) { return sapi->func; } +/* Remove all the APIs registered by the specified module. Usually you + * want this when the module is going to be unloaded. This function + * assumes that's caller responsibility to make sure the APIs are not + * used by other modules. + * + * The number of unregistered APIs is returned. */ +int moduleUnregisterSharedAPI(RedisModule *module) { + int count = 0; + dictIterator *di = dictGetSafeIterator(server.sharedapi); + dictEntry *de; + while ((de = dictNext(di)) != NULL) { + const char *apiname = dictGetKey(de); + RedisModuleSharedAPI *sapi = dictGetVal(de); + if (sapi->module == module) { + dictDelete(server.sharedapi,apiname); + zfree(sapi); + count++; + } + } + dictReleaseIterator(di); + return count; +} + /* -------------------------------------------------------------------------- * Modules API internals * -------------------------------------------------------------------------- */ @@ -4843,6 +4866,7 @@ int moduleLoad(const char *path, void **module_argv, int module_argc) { if (onload((void*)&ctx,module_argv,module_argc) == REDISMODULE_ERR) { if (ctx.module) { moduleUnregisterCommands(ctx.module); + moduleUnregisterSharedAPI(ctx.module); moduleFreeModuleStructure(ctx.module); } dlclose(handle); @@ -4880,6 +4904,7 @@ int moduleUnload(sds name) { } moduleUnregisterCommands(module); + moduleUnregisterSharedAPI(module); /* Remove any notification subscribers this module might have */ moduleUnsubscribeNotifications(module); From f25bcefb825b3bd767bb8aa9a781bcf4bfe2aec5 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 20 Dec 2018 17:31:55 +0100 Subject: [PATCH 08/17] Modules shared API: prevent unloading of used modules. --- src/module.c | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/module.c b/src/module.c index eafdd81f6..4f0e5b126 100644 --- a/src/module.c +++ b/src/module.c @@ -4896,11 +4896,12 @@ int moduleUnload(sds name) { if (module == NULL) { errno = ENOENT; return REDISMODULE_ERR; - } - - if (listLength(module->types)) { + } else if (listLength(module->types)) { errno = EBUSY; return REDISMODULE_ERR; + } else if (listLength(module->usedby)) { + errno = EPERM; + return REDISMODULE_ERR; } moduleUnregisterCommands(module); @@ -4966,7 +4967,12 @@ NULL errmsg = "no such module with that name"; break; case EBUSY: - errmsg = "the module exports one or more module-side data types, can't unload"; + errmsg = "the module exports one or more module-side data " + "types, can't unload"; + break; + case EPERM: + errmsg = "the module exports APIs used by other modules. " + "Please unload them first and try again"; break; default: errmsg = "operation not possible."; From 692f2297b90f9511e4b71e699157c1e737f026fe Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 20 Dec 2018 17:40:55 +0100 Subject: [PATCH 09/17] Modules shared API: also unregister the module as user. --- src/module.c | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/src/module.c b/src/module.c index 4f0e5b126..7bb146954 100644 --- a/src/module.c +++ b/src/module.c @@ -4723,6 +4723,27 @@ int moduleUnregisterSharedAPI(RedisModule *module) { return count; } +/* Remove the specified module as an user of APIs of ever other module. + * This is usually called when a module is unloaded. + * + * Returns the number of modules this module was using APIs from. */ +int moduleUnregisterUsedAPI(RedisModule *module) { + listIter li; + listNode *ln; + int count = 0; + + listRewind(module->using,&li); + while((ln = listNext(&li))) { + RedisModule *used = ln->value; + listNode *ln = listSearchKey(used->usedby,module); + if (ln) { + listDelNode(module->using,ln); + count++; + } + } + return count; +} + /* -------------------------------------------------------------------------- * Modules API internals * -------------------------------------------------------------------------- */ @@ -4867,6 +4888,7 @@ int moduleLoad(const char *path, void **module_argv, int module_argc) { if (ctx.module) { moduleUnregisterCommands(ctx.module); moduleUnregisterSharedAPI(ctx.module); + moduleUnregisterUsedAPI(ctx.module); moduleFreeModuleStructure(ctx.module); } dlclose(handle); @@ -4906,6 +4928,7 @@ int moduleUnload(sds name) { moduleUnregisterCommands(module); moduleUnregisterSharedAPI(module); + moduleUnregisterUsedAPI(module); /* Remove any notification subscribers this module might have */ moduleUnsubscribeNotifications(module); From b91f7656fd8b640b8bbde2b65b65354ebc10f9dc Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 20 Dec 2018 17:44:51 +0100 Subject: [PATCH 10/17] Modules shared API: export new core APIs. --- src/module.c | 2 ++ src/redismodule.h | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/src/module.c b/src/module.c index 7bb146954..f2582193c 100644 --- a/src/module.c +++ b/src/module.c @@ -5184,4 +5184,6 @@ void moduleRegisterCoreAPI(void) { REGISTER_API(DictPrev); REGISTER_API(DictCompareC); REGISTER_API(DictCompare); + REGISTER_API(ExportSharedAPI); + REGISTER_API(GetSharedAPI); } diff --git a/src/redismodule.h b/src/redismodule.h index d18c38881..3a7da18fe 100644 --- a/src/redismodule.h +++ b/src/redismodule.h @@ -332,6 +332,8 @@ void REDISMODULE_API_FUNC(RedisModule_GetRandomBytes)(unsigned char *dst, size_t void REDISMODULE_API_FUNC(RedisModule_GetRandomHexChars)(char *dst, size_t len); void REDISMODULE_API_FUNC(RedisModule_SetDisconnectCallback)(RedisModuleBlockedClient *bc, RedisModuleDisconnectFunc callback); void REDISMODULE_API_FUNC(RedisModule_SetClusterFlags)(RedisModuleCtx *ctx, uint64_t flags); +int REDISMODULE_API_FUNC(RedisModule_ExportSharedAPI)(RedisModuleCtx *ctx, const char *apiname, void *func); +void *REDISMODULE_API_FUNC(RedisModule_GetSharedAPI)(RedisModuleCtx *ctx, const char *apiname); #endif /* This is included inline inside each Redis module. */ @@ -492,6 +494,8 @@ static int RedisModule_Init(RedisModuleCtx *ctx, const char *name, int ver, int REDISMODULE_GET_API(GetRandomBytes); REDISMODULE_GET_API(GetRandomHexChars); REDISMODULE_GET_API(SetClusterFlags); + REDISMODULE_GET_API(ExportSharedAPI); + REDISMODULE_GET_API(GetSharedAPI); #endif if (RedisModule_IsModuleNameBusy && RedisModule_IsModuleNameBusy(name)) return REDISMODULE_ERR; From 59b90db74592a7e07e03368004f679bcf562e0d5 Mon Sep 17 00:00:00 2001 From: Guy Benoish Date: Wed, 23 Jan 2019 11:11:57 +0200 Subject: [PATCH 11/17] ZPOP should return an empty array if COUNT=0 --- src/t_zset.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/t_zset.c b/src/t_zset.c index 0427ee887..1c3da1a28 100644 --- a/src/t_zset.c +++ b/src/t_zset.c @@ -3140,7 +3140,10 @@ void genericZpopCommand(client *c, robj **keyv, int keyc, int where, int emitkey if (countarg) { if (getLongFromObjectOrReply(c,countarg,&count,NULL) != C_OK) return; - if (count < 0) count = 1; + if (count <= 0) { + addReplyNullArray(c); + return; + } } /* Check type and break on the first error, otherwise identify candidate. */ From 7050bf5e2ecdb9d037e4f7a8a9c4ce415afa4157 Mon Sep 17 00:00:00 2001 From: Guy Benoish Date: Mon, 28 Jan 2019 17:58:11 +0200 Subject: [PATCH 12/17] Increase string2ld's buffer size (and fix HINCRBYFLOAT) The string representation of `long double` may take up to ~5000 chars (see PR #3745). Before this fix HINCRBYFLOAT would never overflow (since the string could not exceed 256 chars). Now it can. --- src/t_hash.c | 4 ++++ src/util.c | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/t_hash.c b/src/t_hash.c index d8aee6572..bc70e4051 100644 --- a/src/t_hash.c +++ b/src/t_hash.c @@ -615,6 +615,10 @@ void hincrbyfloatCommand(client *c) { } value += incr; + if (isnan(value) || isinf(value)) { + addReplyError(c,"increment would produce NaN or Infinity"); + return; + } char buf[MAX_LONG_DOUBLE_CHARS]; int len = ld2string(buf,sizeof(buf),value,1); diff --git a/src/util.c b/src/util.c index 66d599190..783bcf83b 100644 --- a/src/util.c +++ b/src/util.c @@ -447,7 +447,7 @@ int string2l(const char *s, size_t slen, long *lval) { * a double: no spaces or other characters before or after the string * representing the number are accepted. */ int string2ld(const char *s, size_t slen, long double *dp) { - char buf[256]; + char buf[MAX_LONG_DOUBLE_CHARS]; long double value; char *eptr; From 9cd3b12cdff082e5499d72a7ece3eafee502657b Mon Sep 17 00:00:00 2001 From: Guy Benoish Date: Tue, 12 Feb 2019 14:21:21 +0100 Subject: [PATCH 13/17] Trim SDS free space of retained module strings In some cases processMultibulkBuffer uses sdsMakeRoomFor to expand the querybuf, but later in some cases it uses that query buffer as is for an argv element (see "Optimization"), which means that the sds in argv may have a lot of wasted space, and then in case modules keep that argv RedisString inside their data structure, this space waste will remain for long (until restarted from rdb). --- src/module.c | 11 +++++++++++ src/object.c | 17 ++++++++++++----- src/sds.c | 4 ++++ src/server.h | 1 + 4 files changed, 28 insertions(+), 5 deletions(-) diff --git a/src/module.c b/src/module.c index 81982ba76..5d3451916 100644 --- a/src/module.c +++ b/src/module.c @@ -509,6 +509,17 @@ void RedisModuleCommandDispatcher(client *c) { cp->func(&ctx,(void**)c->argv,c->argc); moduleHandlePropagationAfterCommandCallback(&ctx); moduleFreeContext(&ctx); + + /* In some cases processMultibulkBuffer uses sdsMakeRoomFor to + * expand the querybuf, but later in some cases it uses that query + * buffer as is for an argv element (see "Optimization"), which means + * that the sds in argv may have a lot of wasted space, and then in case + * modules keep that argv RedisString inside their data structure, this + * space waste will remain for long (until restarted from rdb). */ + for (int i = 0; i < c->argc; i++) { + if (c->argv[i]->refcount > 1) + trimStringObjectIfNeeded(c->argv[i]); + } } /* This function returns the list of keys, with the same interface as the diff --git a/src/object.c b/src/object.c index ec0bd02ee..42c247b87 100644 --- a/src/object.c +++ b/src/object.c @@ -415,6 +415,17 @@ int isObjectRepresentableAsLongLong(robj *o, long long *llval) { } } +void trimStringObjectIfNeeded(robj *o) { + /* Optimize the SDS string inside the string object to require + * little space, in case there is more than 10% of free space + * at the end of the SDS string. */ + if (o->encoding == OBJ_ENCODING_RAW && + sdsavail(o->ptr) > sdslen(o->ptr)/10) + { + o->ptr = sdsRemoveFreeSpace(o->ptr); + } +} + /* Try to encode a string object in order to save space */ robj *tryObjectEncoding(robj *o) { long value; @@ -484,11 +495,7 @@ robj *tryObjectEncoding(robj *o) { * We do that only for relatively large strings as this branch * is only entered if the length of the string is greater than * OBJ_ENCODING_EMBSTR_SIZE_LIMIT. */ - if (o->encoding == OBJ_ENCODING_RAW && - sdsavail(s) > len/10) - { - o->ptr = sdsRemoveFreeSpace(o->ptr); - } + trimStringObjectIfNeeded(o); /* Return the original object. */ return o; diff --git a/src/sds.c b/src/sds.c index 330c955e8..cd60946bd 100644 --- a/src/sds.c +++ b/src/sds.c @@ -257,8 +257,12 @@ sds sdsRemoveFreeSpace(sds s) { char type, oldtype = s[-1] & SDS_TYPE_MASK; int hdrlen, oldhdrlen = sdsHdrSize(oldtype); size_t len = sdslen(s); + size_t avail = sdsavail(s); sh = (char*)s-oldhdrlen; + /* Return ASAP if there is no space left. */ + if (avail == 0) return s; + /* Check what would be the minimum SDS header that is just good enough to * fit this string. */ type = sdsReqType(len); diff --git a/src/server.h b/src/server.h index a396e1cf7..34b4cd8d5 100644 --- a/src/server.h +++ b/src/server.h @@ -1656,6 +1656,7 @@ int compareStringObjects(robj *a, robj *b); int collateStringObjects(robj *a, robj *b); int equalStringObjects(robj *a, robj *b); unsigned long long estimateObjectIdleTime(robj *o); +void trimStringObjectIfNeeded(robj *o); #define sdsEncodedObject(objptr) (objptr->encoding == OBJ_ENCODING_RAW || objptr->encoding == OBJ_ENCODING_EMBSTR) /* Synchronous I/O with timeout */ From 24300677e7172c9081a865edbd417a89b7987042 Mon Sep 17 00:00:00 2001 From: Guy Benoish Date: Thu, 14 Mar 2019 12:11:16 +0100 Subject: [PATCH 14/17] Fix mismatching keyspace notification classes --- src/geo.c | 2 +- src/t_list.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/geo.c b/src/geo.c index 91a0421f5..826d11ff5 100644 --- a/src/geo.c +++ b/src/geo.c @@ -659,7 +659,7 @@ void georadiusGeneric(client *c, int flags) { zsetConvertToZiplistIfNeeded(zobj,maxelelen); setKey(c->db,storekey,zobj); decrRefCount(zobj); - notifyKeyspaceEvent(NOTIFY_LIST,"georadiusstore",storekey, + notifyKeyspaceEvent(NOTIFY_ZSET,"georadiusstore",storekey, c->db->id); server.dirty += returned_items; } else if (dbDelete(c->db,storekey)) { diff --git a/src/t_list.c b/src/t_list.c index 451ffb4b5..45d2e3317 100644 --- a/src/t_list.c +++ b/src/t_list.c @@ -520,7 +520,7 @@ void lremCommand(client *c) { if (removed) { signalModifiedKey(c->db,c->argv[1]); - notifyKeyspaceEvent(NOTIFY_GENERIC,"lrem",c->argv[1],c->db->id); + notifyKeyspaceEvent(NOTIFY_LIST,"lrem",c->argv[1],c->db->id); } if (listTypeLength(subject) == 0) { From 9c504573f18ac6ad15360daa3d0b44179e693ccb Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 14 Mar 2019 12:47:36 +0100 Subject: [PATCH 15/17] Improve comments after merging #5834. --- src/module.c | 15 ++++++++++----- src/object.c | 7 ++++--- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/src/module.c b/src/module.c index 28a4d3e64..e69d3dc61 100644 --- a/src/module.c +++ b/src/module.c @@ -523,12 +523,17 @@ void RedisModuleCommandDispatcher(client *c) { moduleFreeContext(&ctx); /* In some cases processMultibulkBuffer uses sdsMakeRoomFor to - * expand the querybuf, but later in some cases it uses that query - * buffer as is for an argv element (see "Optimization"), which means - * that the sds in argv may have a lot of wasted space, and then in case - * modules keep that argv RedisString inside their data structure, this - * space waste will remain for long (until restarted from rdb). */ + * expand the query buffer, and in order to avoid a big object copy + * the query buffer SDS may be used directly as the SDS string backing + * the client argument vectors: sometimes this will result in the SDS + * string having unused space at the end. Later if a module takes ownership + * of the RedisString, such space will be wasted forever. Inside the + * Redis core this is not a problem because tryObjectEncoding() is called + * before storing strings in the key space. Here we need to do it + * for the module. */ for (int i = 0; i < c->argc; i++) { + /* Only do the work if the module took ownership of the object: + * in that case the refcount is no longer 1. */ if (c->argv[i]->refcount > 1) trimStringObjectIfNeeded(c->argv[i]); } diff --git a/src/object.c b/src/object.c index 42c247b87..24e99d191 100644 --- a/src/object.c +++ b/src/object.c @@ -415,10 +415,11 @@ int isObjectRepresentableAsLongLong(robj *o, long long *llval) { } } +/* Optimize the SDS string inside the string object to require little space, + * in case there is more than 10% of free space at the end of the SDS + * string. This happens because SDS strings tend to overallocate to avoid + * wasting too much time in allocations when appending to the string. */ void trimStringObjectIfNeeded(robj *o) { - /* Optimize the SDS string inside the string object to require - * little space, in case there is more than 10% of free space - * at the end of the SDS string. */ if (o->encoding == OBJ_ENCODING_RAW && sdsavail(o->ptr) > sdslen(o->ptr)/10) { From e8568bb34189cf45b3816a87c7c24c996e1cd827 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 14 Mar 2019 17:06:59 +0100 Subject: [PATCH 16/17] Fix objectSetLRUOrLFU() when LFU underflows. --- src/object.c | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/src/object.c b/src/object.c index 24e99d191..234e11f8a 100644 --- a/src/object.c +++ b/src/object.c @@ -1199,7 +1199,7 @@ sds getMemoryDoctorReport(void) { /* Set the object LRU/LFU depending on server.maxmemory_policy. * The lfu_freq arg is only relevant if policy is MAXMEMORY_FLAG_LFU. - * The lru_idle and lru_clock args are only relevant if policy + * The lru_idle and lru_clock args are only relevant if policy * is MAXMEMORY_FLAG_LRU. * Either or both of them may be <0, in that case, nothing is set. */ void objectSetLRUOrLFU(robj *val, long long lfu_freq, long long lru_idle, @@ -1210,16 +1210,20 @@ void objectSetLRUOrLFU(robj *val, long long lfu_freq, long long lru_idle, val->lru = (LFUGetTimeInMinutes()<<8) | lfu_freq; } } else if (lru_idle >= 0) { - /* Serialized LRU idle time is in seconds. Scale + /* Provided LRU idle time is in seconds. Scale * according to the LRU clock resolution this Redis * instance was compiled with (normally 1000 ms, so the * below statement will expand to lru_idle*1000/1000. */ lru_idle = lru_idle*1000/LRU_CLOCK_RESOLUTION; - val->lru = lru_clock - lru_idle; - /* If the lru field overflows (since LRU it is a wrapping - * clock), the best we can do is to provide the maximum - * representable idle time. */ - if (val->lru < 0) val->lru = lru_clock+1; + long lru_abs = lru_clock - lru_idle; /* Absolute access time. */ + /* If the LRU field underflows (since LRU it is a wrapping + * clock), the best we can do is to provide a large enough LRU + * that is half-way in the circlular LRU clock we use: this way + * the computed idle time for this object will stay high for quite + * some time. */ + if (lru_abs < 0) + lru_abs = (lru_clock+(LRU_CLOCK_MAX/2)) % LRU_CLOCK_MAX; + val->lru = lru_abs; } } From 5199f1dc584b72c2c6c7afab19cca6bfab71c5d3 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 14 Mar 2019 17:51:14 +0100 Subject: [PATCH 17/17] Fix ZPOP return type when COUNT=0. Related to #5799. --- src/t_zset.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/t_zset.c b/src/t_zset.c index 0daa6d643..fb7078abd 100644 --- a/src/t_zset.c +++ b/src/t_zset.c @@ -3144,7 +3144,7 @@ void genericZpopCommand(client *c, robj **keyv, int keyc, int where, int emitkey if (getLongFromObjectOrReply(c,countarg,&count,NULL) != C_OK) return; if (count <= 0) { - addReplyNullArray(c); + addReply(c,shared.emptyarray); return; } }