diff --git a/src/config.c b/src/config.c index da548cba9..fe563e7ed 100644 --- a/src/config.c +++ b/src/config.c @@ -1103,8 +1103,8 @@ void configSetCommand(client *c) { int soft_seconds; class = getClientTypeByName(v[j]); - hard = strtoll(v[j+1],NULL,10); - soft = strtoll(v[j+2],NULL,10); + hard = memtoll(v[j+1],NULL); + soft = memtoll(v[j+2],NULL); soft_seconds = strtoll(v[j+3],NULL,10); server.client_obuf_limits[class].hard_limit_bytes = hard; diff --git a/src/geo.c b/src/geo.c index f04e83f5e..7fc52d68d 100644 --- a/src/geo.c +++ b/src/geo.c @@ -659,7 +659,7 @@ void georadiusGeneric(client *c, int flags) { zsetConvertToZiplistIfNeeded(zobj,maxelelen); setKey(c->db,storekey,zobj); decrRefCount(zobj); - notifyKeyspaceEvent(NOTIFY_LIST,"georadiusstore",storekey, + notifyKeyspaceEvent(NOTIFY_ZSET,"georadiusstore",storekey, c->db->id); server.dirty += returned_items; } else if (dbDelete(c->db,storekey)) { diff --git a/src/module.c b/src/module.c index aa4b5de26..fda6bcf66 100644 --- a/src/module.c +++ b/src/module.c @@ -47,9 +47,21 @@ struct RedisModule { int ver; /* Module version. We use just progressive integers. */ int apiver; /* Module API version as requested during initialization.*/ list *types; /* Module data types. */ + list *usedby; /* List of modules using APIs from this one. */ + list *using; /* List of modules we use some APIs of. */ }; typedef struct RedisModule RedisModule; +/* This represents a shared API. Shared APIs will be used to populate + * the server.sharedapi dictionary, mapping names of APIs exported by + * modules for other modules to use, to their structure specifying the + * function pointer that can be called. */ +struct RedisModuleSharedAPI { + void *func; + RedisModule *module; +}; +typedef struct RedisModuleSharedAPI RedisModuleSharedAPI; + static dict *modules; /* Hash table of modules. SDS -> RedisModule ptr.*/ /* Entries in the context->amqueue array, representing objects to free @@ -510,6 +522,22 @@ void RedisModuleCommandDispatcher(client *c) { cp->func(&ctx,(void**)c->argv,c->argc); moduleHandlePropagationAfterCommandCallback(&ctx); moduleFreeContext(&ctx); + + /* In some cases processMultibulkBuffer uses sdsMakeRoomFor to + * expand the query buffer, and in order to avoid a big object copy + * the query buffer SDS may be used directly as the SDS string backing + * the client argument vectors: sometimes this will result in the SDS + * string having unused space at the end. Later if a module takes ownership + * of the RedisString, such space will be wasted forever. Inside the + * Redis core this is not a problem because tryObjectEncoding() is called + * before storing strings in the key space. Here we need to do it + * for the module. */ + for (int i = 0; i < c->argc; i++) { + /* Only do the work if the module took ownership of the object: + * in that case the refcount is no longer 1. */ + if (c->argv[i]->refcount > 1) + trimStringObjectIfNeeded(c->argv[i]); + } } /* This function returns the list of keys, with the same interface as the @@ -702,6 +730,8 @@ void RM_SetModuleAttribs(RedisModuleCtx *ctx, const char *name, int ver, int api module->ver = ver; module->apiver = apiver; module->types = listCreate(); + module->usedby = listCreate(); + module->using = listCreate(); ctx->module = module; } @@ -3430,6 +3460,8 @@ void RM_LogRaw(RedisModule *module, const char *levelstr, const char *fmt, va_li else if (!strcasecmp(levelstr,"warning")) level = LL_WARNING; else level = LL_VERBOSE; /* Default. */ + if (level < server.verbosity) return; + name_len = snprintf(msg, sizeof(msg),"<%s> ", module->name); vsnprintf(msg + name_len, sizeof(msg) - name_len, fmt, ap); serverLogRaw(level,msg); @@ -4653,6 +4685,121 @@ void RM_GetRandomHexChars(char *dst, size_t len) { getRandomHexChars(dst,len); } +/* -------------------------------------------------------------------------- + * Modules API exporting / importing + * -------------------------------------------------------------------------- */ + +/* This function is called by a module in order to export some API with a + * given name. Other modules will be able to use this API by calling the + * symmetrical function RM_GetSharedAPI() and casting the return value to + * the right function pointer. + * + * The function will return REDISMODULE_OK if the name is not already taken, + * otherwise REDISMODULE_ERR will be returned and no operation will be + * performed. + * + * IMPORTANT: the apiname argument should be a string literal with static + * lifetime. The API relies on the fact that it will always be valid in + * the future. */ +int RM_ExportSharedAPI(RedisModuleCtx *ctx, const char *apiname, void *func) { + RedisModuleSharedAPI *sapi = zmalloc(sizeof(*sapi), MALLOC_LOCAL); + sapi->module = ctx->module; + sapi->func = func; + if (dictAdd(server.sharedapi, (char*)apiname, sapi) != DICT_OK) { + zfree(sapi); + return REDISMODULE_ERR; + } + return REDISMODULE_OK; +} + +/* Request an exported API pointer. The return value is just a void pointer + * that the caller of this function will be required to cast to the right + * function pointer, so this is a private contract between modules. + * + * If the requested API is not available then NULL is returned. Because + * modules can be loaded at different times with different order, this + * function calls should be put inside some module generic API registering + * step, that is called every time a module attempts to execute a + * command that requires external APIs: if some API cannot be resolved, the + * command should return an error. + * + * Here is an exmaple: + * + * int ... myCommandImplementation() { + * if (getExternalAPIs() == 0) { + * reply with an error here if we cannot have the APIs + * } + * // Use the API: + * myFunctionPointer(foo); + * } + * + * And the function registerAPI() is: + * + * int getExternalAPIs(void) { + * static int api_loaded = 0; + * if (api_loaded != 0) return 1; // APIs already resolved. + * + * myFunctionPointer = RedisModule_GetOtherModuleAPI("..."); + * if (myFunctionPointer == NULL) return 0; + * + * return 1; + * } + */ +void *RM_GetSharedAPI(RedisModuleCtx *ctx, const char *apiname) { + dictEntry *de = dictFind(server.sharedapi, apiname); + if (de == NULL) return NULL; + RedisModuleSharedAPI *sapi = dictGetVal(de); + if (listSearchKey(sapi->module->usedby,ctx->module) == NULL) { + listAddNodeTail(sapi->module->usedby,ctx->module); + listAddNodeTail(ctx->module->using,sapi->module); + } + return sapi->func; +} + +/* Remove all the APIs registered by the specified module. Usually you + * want this when the module is going to be unloaded. This function + * assumes that's caller responsibility to make sure the APIs are not + * used by other modules. + * + * The number of unregistered APIs is returned. */ +int moduleUnregisterSharedAPI(RedisModule *module) { + int count = 0; + dictIterator *di = dictGetSafeIterator(server.sharedapi); + dictEntry *de; + while ((de = dictNext(di)) != NULL) { + const char *apiname = dictGetKey(de); + RedisModuleSharedAPI *sapi = dictGetVal(de); + if (sapi->module == module) { + dictDelete(server.sharedapi,apiname); + zfree(sapi); + count++; + } + } + dictReleaseIterator(di); + return count; +} + +/* Remove the specified module as an user of APIs of ever other module. + * This is usually called when a module is unloaded. + * + * Returns the number of modules this module was using APIs from. */ +int moduleUnregisterUsedAPI(RedisModule *module) { + listIter li; + listNode *ln; + int count = 0; + + listRewind(module->using,&li); + while((ln = listNext(&li))) { + RedisModule *used = ln->value; + listNode *ln = listSearchKey(used->usedby,module); + if (ln) { + listDelNode(module->using,ln); + count++; + } + } + return count; +} + /* -------------------------------------------------------------------------- * Modules API internals * -------------------------------------------------------------------------- */ @@ -4798,6 +4945,8 @@ int moduleLoad(const char *path, void **module_argv, int module_argc) { if (onload((void*)&ctx,module_argv,module_argc) == REDISMODULE_ERR) { if (ctx.module) { moduleUnregisterCommands(ctx.module); + moduleUnregisterSharedAPI(ctx.module); + moduleUnregisterUsedAPI(ctx.module); moduleFreeModuleStructure(ctx.module); } dlclose(handle); @@ -4827,14 +4976,17 @@ int moduleUnload(sds name) { if (module == NULL) { errno = ENOENT; return REDISMODULE_ERR; - } - - if (listLength(module->types)) { + } else if (listLength(module->types)) { errno = EBUSY; return REDISMODULE_ERR; + } else if (listLength(module->usedby)) { + errno = EPERM; + return REDISMODULE_ERR; } moduleUnregisterCommands(module); + moduleUnregisterSharedAPI(module); + moduleUnregisterUsedAPI(module); /* Remove any notification subscribers this module might have */ moduleUnsubscribeNotifications(module); @@ -4915,7 +5067,12 @@ NULL errmsg = "no such module with that name"; break; case EBUSY: - errmsg = "the module exports one or more module-side data types, can't unload"; + errmsg = "the module exports one or more module-side data " + "types, can't unload"; + break; + case EPERM: + errmsg = "the module exports APIs used by other modules. " + "Please unload them first and try again"; break; default: errmsg = "operation not possible."; @@ -4940,6 +5097,7 @@ size_t moduleCount(void) { * file so that's easy to seek it to add new entries. */ void moduleRegisterCoreAPI(void) { server.moduleapi = dictCreate(&moduleAPIDictType,NULL); + server.sharedapi = dictCreate(&moduleAPIDictType,NULL); REGISTER_API(Alloc); REGISTER_API(Calloc); REGISTER_API(Realloc); @@ -5090,4 +5248,6 @@ void moduleRegisterCoreAPI(void) { REGISTER_API(DictPrev); REGISTER_API(DictCompareC); REGISTER_API(DictCompare); + REGISTER_API(ExportSharedAPI); + REGISTER_API(GetSharedAPI); } diff --git a/src/object.c b/src/object.c index 600dbfbc9..7112e0593 100644 --- a/src/object.c +++ b/src/object.c @@ -417,6 +417,18 @@ int isObjectRepresentableAsLongLong(robj *o, long long *llval) { } } +/* Optimize the SDS string inside the string object to require little space, + * in case there is more than 10% of free space at the end of the SDS + * string. This happens because SDS strings tend to overallocate to avoid + * wasting too much time in allocations when appending to the string. */ +void trimStringObjectIfNeeded(robj *o) { + if (o->encoding == OBJ_ENCODING_RAW && + sdsavail(ptrFromObj(o)) > sdslen(ptrFromObj(o))/10) + { + o->m_ptr = sdsRemoveFreeSpace(ptrFromObj(o)); + } +} + /* Try to encode a string object in order to save space */ robj *tryObjectEncoding(robj *o) { long value; @@ -486,11 +498,7 @@ robj *tryObjectEncoding(robj *o) { * We do that only for relatively large strings as this branch * is only entered if the length of the string is greater than * OBJ_ENCODING_EMBSTR_SIZE_LIMIT. */ - if (o->encoding == OBJ_ENCODING_RAW && - sdsavail(s) > len/10) - { - o->m_ptr = sdsRemoveFreeSpace(ptrFromObj(o)); - } + trimStringObjectIfNeeded(o); /* Return the original object. */ return o; @@ -1197,7 +1205,7 @@ sds getMemoryDoctorReport(void) { /* Set the object LRU/LFU depending on server.maxmemory_policy. * The lfu_freq arg is only relevant if policy is MAXMEMORY_FLAG_LFU. - * The lru_idle and lru_clock args are only relevant if policy + * The lru_idle and lru_clock args are only relevant if policy * is MAXMEMORY_FLAG_LRU. * Either or both of them may be <0, in that case, nothing is set. */ void objectSetLRUOrLFU(robj *val, long long lfu_freq, long long lru_idle, @@ -1208,16 +1216,20 @@ void objectSetLRUOrLFU(robj *val, long long lfu_freq, long long lru_idle, val->lru = (LFUGetTimeInMinutes()<<8) | lfu_freq; } } else if (lru_idle >= 0) { - /* Serialized LRU idle time is in seconds. Scale + /* Provided LRU idle time is in seconds. Scale * according to the LRU clock resolution this Redis * instance was compiled with (normally 1000 ms, so the * below statement will expand to lru_idle*1000/1000. */ lru_idle = lru_idle*1000/LRU_CLOCK_RESOLUTION; - val->lru = lru_clock - lru_idle; - /* If the lru field overflows (since LRU it is a wrapping - * clock), the best we can do is to provide the maximum - * representable idle time. */ - if (val->lru < 0) val->lru = lru_clock+1; + long lru_abs = lru_clock - lru_idle; /* Absolute access time. */ + /* If the LRU field underflows (since LRU it is a wrapping + * clock), the best we can do is to provide a large enough LRU + * that is half-way in the circlular LRU clock we use: this way + * the computed idle time for this object will stay high for quite + * some time. */ + if (lru_abs < 0) + lru_abs = (lru_clock+(LRU_CLOCK_MAX/2)) % LRU_CLOCK_MAX; + val->lru = lru_abs; } } diff --git a/src/redismodule.h b/src/redismodule.h index 540f8e3db..272da08df 100644 --- a/src/redismodule.h +++ b/src/redismodule.h @@ -335,6 +335,8 @@ void REDISMODULE_API_FUNC(RedisModule_GetRandomBytes)(unsigned char *dst, size_t void REDISMODULE_API_FUNC(RedisModule_GetRandomHexChars)(char *dst, size_t len); void REDISMODULE_API_FUNC(RedisModule_SetDisconnectCallback)(RedisModuleBlockedClient *bc, RedisModuleDisconnectFunc callback); void REDISMODULE_API_FUNC(RedisModule_SetClusterFlags)(RedisModuleCtx *ctx, uint64_t flags); +int REDISMODULE_API_FUNC(RedisModule_ExportSharedAPI)(RedisModuleCtx *ctx, const char *apiname, void *func); +void *REDISMODULE_API_FUNC(RedisModule_GetSharedAPI)(RedisModuleCtx *ctx, const char *apiname); #endif /* This is included inline inside each Redis module. */ @@ -495,6 +497,8 @@ static int RedisModule_Init(RedisModuleCtx *ctx, const char *name, int ver, int REDISMODULE_GET_API(GetRandomBytes); REDISMODULE_GET_API(GetRandomHexChars); REDISMODULE_GET_API(SetClusterFlags); + REDISMODULE_GET_API(ExportSharedAPI); + REDISMODULE_GET_API(GetSharedAPI); #endif if (RedisModule_IsModuleNameBusy && RedisModule_IsModuleNameBusy(name)) return REDISMODULE_ERR; diff --git a/src/sds.c b/src/sds.c index dac1f281f..9c0da50cb 100644 --- a/src/sds.c +++ b/src/sds.c @@ -259,8 +259,12 @@ sds sdsRemoveFreeSpace(sds s) { char type, oldtype = s[-1] & SDS_TYPE_MASK; int hdrlen, oldhdrlen = sdsHdrSize(oldtype); size_t len = sdslen(s); + size_t avail = sdsavail(s); sh = (char*)s-oldhdrlen; + /* Return ASAP if there is no space left. */ + if (avail == 0) return s; + /* Check what would be the minimum SDS header that is just good enough to * fit this string. */ type = sdsReqType(len); diff --git a/src/server.h b/src/server.h index b4048c4ed..212ba9c81 100644 --- a/src/server.h +++ b/src/server.h @@ -1087,7 +1087,9 @@ struct redisServer { size_t initial_memory_usage; /* Bytes used after initialization. */ int always_show_logo; /* Show logo even for non-stdout logging. */ /* Modules */ - dict *moduleapi; /* Exported APIs dictionary for modules. */ + dict *moduleapi; /* Exported core APIs dictionary for modules. */ + dict *sharedapi; /* Like moduleapi but containing the APIs that + modules share with each other. */ list *loadmodule_queue; /* List of modules to load at startup. */ int module_blocked_pipe[2]; /* Pipe used to awake the event loop if a client blocked on a module command needs @@ -1740,6 +1742,7 @@ int compareStringObjects(robj *a, robj *b); int collateStringObjects(robj *a, robj *b); int equalStringObjects(robj *a, robj *b); unsigned long long estimateObjectIdleTime(robj *o); +void trimStringObjectIfNeeded(robj *o); #define sdsEncodedObject(objptr) (objptr->encoding == OBJ_ENCODING_RAW || objptr->encoding == OBJ_ENCODING_EMBSTR) /* Synchronous I/O with timeout */ diff --git a/src/t_hash.c b/src/t_hash.c index f5d31d966..8c418314b 100644 --- a/src/t_hash.c +++ b/src/t_hash.c @@ -615,6 +615,10 @@ void hincrbyfloatCommand(client *c) { } value += incr; + if (isnan(value) || isinf(value)) { + addReplyError(c,"increment would produce NaN or Infinity"); + return; + } char buf[MAX_LONG_DOUBLE_CHARS]; int len = ld2string(buf,sizeof(buf),value,1); diff --git a/src/t_list.c b/src/t_list.c index c7350887f..3c14f44e3 100644 --- a/src/t_list.c +++ b/src/t_list.c @@ -520,7 +520,7 @@ void lremCommand(client *c) { if (removed) { signalModifiedKey(c->db,c->argv[1]); - notifyKeyspaceEvent(NOTIFY_GENERIC,"lrem",c->argv[1],c->db->id); + notifyKeyspaceEvent(NOTIFY_LIST,"lrem",c->argv[1],c->db->id); } if (listTypeLength(subject) == 0) { diff --git a/src/t_zset.c b/src/t_zset.c index 7fbcf3bbf..17be22433 100644 --- a/src/t_zset.c +++ b/src/t_zset.c @@ -2906,7 +2906,10 @@ void genericZrangebylexCommand(client *c, int reverse) { while (remaining) { if (remaining >= 3 && !strcasecmp(ptrFromObj(c->argv[pos]),"limit")) { if ((getLongFromObjectOrReply(c, c->argv[pos+1], &offset, NULL) != C_OK) || - (getLongFromObjectOrReply(c, c->argv[pos+2], &limit, NULL) != C_OK)) return; + (getLongFromObjectOrReply(c, c->argv[pos+2], &limit, NULL) != C_OK)) { + zslFreeLexRange(&range); + return; + } pos += 3; remaining -= 3; } else { zslFreeLexRange(&range); @@ -3140,7 +3143,10 @@ void genericZpopCommand(client *c, robj **keyv, int keyc, int where, int emitkey if (countarg) { if (getLongFromObjectOrReply(c,countarg,&count,NULL) != C_OK) return; - if (count < 0) count = 1; + if (count <= 0) { + addReply(c,shared.emptyarray); + return; + } } /* Check type and break on the first error, otherwise identify candidate. */ diff --git a/src/util.c b/src/util.c index 66d599190..783bcf83b 100644 --- a/src/util.c +++ b/src/util.c @@ -447,7 +447,7 @@ int string2l(const char *s, size_t slen, long *lval) { * a double: no spaces or other characters before or after the string * representing the number are accepted. */ int string2ld(const char *s, size_t slen, long double *dp) { - char buf[256]; + char buf[MAX_LONG_DOUBLE_CHARS]; long double value; char *eptr;