/* * Copyright (c) 2016, Redis Ltd. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * * Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * * Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * Neither the name of Redis nor the names of its contributors may be used * to endorse or promote products derived from this software without * specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ /* -------------------------------------------------------------------------- * Modules API documentation information * * The comments in this file are used to generate the API documentation on the * website. * * Each function starting with VM_ and preceded by a block comment is included * in the API documentation. To hide a VM_ function, put a blank line between * the comment and the function definition or put the comment inside the * function body. * * The functions are divided into sections. Each section is preceded by a * documentation block, which is comment block starting with a markdown level 2 * heading, i.e. a line starting with ##, on the first line of the comment block * (with the exception of a ----- line which can appear first). Other comment * blocks, which are not intended for the modules API user, such as this comment * block, do NOT start with a markdown level 2 heading, so they are included in * the generated a API documentation. * * The documentation comments may contain markdown formatting. Some automatic * replacements are done, such as the replacement of RM with ValkeyModule in * function names. For details, see the script src/modules/gendoc.rb. * -------------------------------------------------------------------------- */ #include "server.h" #include "cluster.h" #include "commandlog.h" #include "rdb.h" #include "monotonic.h" #include "script.h" #include "call_reply.h" #include "hdr_histogram.h" #include "crc16_slottable.h" #include "valkeymodule.h" #include "io_threads.h" #include "module.h" #include "scripting_engine.h" #include #include #include #include #include /* -------------------------------------------------------------------------- * Private data structures used by the modules system. Those are data * structures that are never exposed to Modules, if not as void * pointers that have an API the module can call with them) * -------------------------------------------------------------------------- */ struct moduleLoadQueueEntry { sds path; int argc; robj **argv; }; struct ValkeyModuleInfoCtx { struct ValkeyModule *module; dict *requested_sections; sds info; /* info string we collected so far */ int sections; /* number of sections we collected so far */ int in_section; /* indication if we're in an active section or not */ int in_dict_field; /* indication that we're currently appending to a dict */ }; /* This represents a shared API. Shared APIs will be used to populate * the server.sharedapi dictionary, mapping names of APIs exported by * modules for other modules to use, to their structure specifying the * function pointer that can be called. */ struct ValkeyModuleSharedAPI { void *func; ValkeyModule *module; }; typedef struct ValkeyModuleSharedAPI ValkeyModuleSharedAPI; dict *modules; /* Hash table of modules. SDS -> ValkeyModule ptr.*/ /* Entries in the context->amqueue array, representing objects to free * when the callback returns. */ struct AutoMemEntry { void *ptr; int type; }; /* AutoMemEntry type field values. */ #define VALKEYMODULE_AM_KEY 0 #define VALKEYMODULE_AM_STRING 1 #define VALKEYMODULE_AM_REPLY 2 #define VALKEYMODULE_AM_FREED 3 /* Explicitly freed by user already. */ #define VALKEYMODULE_AM_DICT 4 #define VALKEYMODULE_AM_INFO 5 /* The pool allocator block. Modules can allocate memory via this special * allocator that will automatically release it all once the callback returns. * This means that it can only be used for ephemeral allocations. However * there are two advantages for modules to use this API: * * 1) The memory is automatically released when the callback returns. * 2) This allocator is faster for many small allocations since whole blocks * are allocated, and small pieces returned to the caller just advancing * the index of the allocation. * * Allocations are always rounded to the size of the void pointer in order * to always return aligned memory chunks. */ #define VALKEYMODULE_POOL_ALLOC_MIN_SIZE (1024 * 8) #define VALKEYMODULE_POOL_ALLOC_ALIGN (sizeof(void *)) typedef struct ValkeyModulePoolAllocBlock { uint32_t size; uint32_t used; struct ValkeyModulePoolAllocBlock *next; char memory[]; } ValkeyModulePoolAllocBlock; /* This structure represents the context in which modules operate. * Most APIs module can access, get a pointer to the context, so that the API * implementation can hold state across calls, or remember what to free after * the call and so forth. * * Note that not all the context structure is always filled with actual values * but only the fields needed in a given context. */ struct ValkeyModuleBlockedClient; struct ValkeyModuleUser; struct ValkeyModuleCtx { void *getapifuncptr; /* NOTE: Must be the first field. */ struct ValkeyModule *module; /* Module reference. */ client *client; /* Client calling a command. */ struct ValkeyModuleBlockedClient *blocked_client; /* Blocked client for thread safe context. */ struct AutoMemEntry *amqueue; /* Auto memory queue of objects to free. */ int amqueue_len; /* Number of slots in amqueue. */ int amqueue_used; /* Number of used slots in amqueue. */ int flags; /* VALKEYMODULE_CTX_... flags. */ void **postponed_arrays; /* To set with VM_ReplySetArrayLength(). */ int postponed_arrays_count; /* Number of entries in postponed_arrays. */ void *blocked_privdata; /* Privdata set when unblocking a client. */ ValkeyModuleString *blocked_ready_key; /* Key ready when the reply callback gets called for clients blocked on keys. */ /* Used if there is the VALKEYMODULE_CTX_KEYS_POS_REQUEST or * VALKEYMODULE_CTX_CHANNEL_POS_REQUEST flag set. */ getKeysResult *keys_result; struct ValkeyModulePoolAllocBlock *pa_head; long long next_yield_time; const struct ValkeyModuleUser *user; /* ValkeyModuleUser commands executed via VM_Call should be executed as, if set */ }; typedef struct ValkeyModuleCtx ValkeyModuleCtx; #define VALKEYMODULE_CTX_NONE (0) #define VALKEYMODULE_CTX_AUTO_MEMORY (1 << 0) #define VALKEYMODULE_CTX_KEYS_POS_REQUEST (1 << 1) #define VALKEYMODULE_CTX_BLOCKED_REPLY (1 << 2) #define VALKEYMODULE_CTX_BLOCKED_TIMEOUT (1 << 3) #define VALKEYMODULE_CTX_THREAD_SAFE (1 << 4) #define VALKEYMODULE_CTX_BLOCKED_DISCONNECTED (1 << 5) #define VALKEYMODULE_CTX_TEMP_CLIENT (1 << 6) /* Return client object to the pool \ when the context is destroyed */ #define VALKEYMODULE_CTX_NEW_CLIENT (1 << 7) /* Free client object when the \ context is destroyed */ #define VALKEYMODULE_CTX_CHANNELS_POS_REQUEST (1 << 8) #define VALKEYMODULE_CTX_COMMAND (1 << 9) /* Context created to serve a command from call() or AOF (which calls cmd->proc directly) */ /* This represents a key opened with VM_OpenKey(). */ struct ValkeyModuleKey { ValkeyModuleCtx *ctx; serverDb *db; robj *key; /* Key name object. */ robj *value; /* Value object, or NULL if the key was not found. */ void *iter; /* Iterator. */ int mode; /* Opening mode. */ union { struct { /* List, use only if value->type == OBJ_LIST */ listTypeEntry entry; /* Current entry in iteration. */ long index; /* Current 0-based index in iteration. */ } list; struct { /* Zset iterator, use only if value->type == OBJ_ZSET */ uint32_t type; /* VALKEYMODULE_ZSET_RANGE_* */ zrangespec rs; /* Score range. */ zlexrangespec lrs; /* Lex range. */ uint32_t start; /* Start pos for positional ranges. */ uint32_t end; /* End pos for positional ranges. */ void *current; /* Zset iterator current node. */ int er; /* Zset iterator end reached flag (true if end was reached). */ } zset; struct { /* Stream, use only if value->type == OBJ_STREAM */ streamID currentid; /* Current entry while iterating. */ int64_t numfieldsleft; /* Fields left to fetch for current entry. */ int signalready; /* Flag that signalKeyAsReady() is needed. */ } stream; } u; }; /* ValkeyModuleKey 'ztype' values. */ #define VALKEYMODULE_ZSET_RANGE_NONE 0 /* This must always be 0. */ #define VALKEYMODULE_ZSET_RANGE_LEX 1 #define VALKEYMODULE_ZSET_RANGE_SCORE 2 #define VALKEYMODULE_ZSET_RANGE_POS 3 /* Function pointer type of a function representing a command inside * a module. */ struct ValkeyModuleBlockedClient; typedef int (*ValkeyModuleCmdFunc)(ValkeyModuleCtx *ctx, void **argv, int argc); typedef int (*ValkeyModuleAuthCallback)(ValkeyModuleCtx *ctx, void *username, void *password, ValkeyModuleString **err); typedef void (*ValkeyModuleDisconnectFunc)(ValkeyModuleCtx *ctx, struct ValkeyModuleBlockedClient *bc); /* This struct holds the information about a command registered by a module.*/ struct ValkeyModuleCommand { struct ValkeyModule *module; ValkeyModuleCmdFunc func; struct serverCommand *serverCmd; }; typedef struct ValkeyModuleCommand ValkeyModuleCommand; #define VALKEYMODULE_REPLYFLAG_NONE 0 #define VALKEYMODULE_REPLYFLAG_TOPARSE (1 << 0) /* Protocol must be parsed. */ #define VALKEYMODULE_REPLYFLAG_NESTED (1 << 1) /* Nested reply object. No proto \ or struct free. */ /* Reply of VM_Call() function. The function is filled in a lazy * way depending on the function called on the reply structure. By default * only the type, proto and protolen are filled. */ typedef struct CallReply ValkeyModuleCallReply; /* Structure to hold the module auth callback & the Module implementing it. */ typedef struct ValkeyModuleAuthCtx { struct ValkeyModule *module; ValkeyModuleAuthCallback auth_cb; } ValkeyModuleAuthCtx; /* Structure representing a blocked client. We get a pointer to such * an object when blocking from modules. */ typedef struct ValkeyModuleBlockedClient { client *client; /* Pointer to the blocked client. or NULL if the client was destroyed during the life of this object. */ ValkeyModule *module; /* Module blocking the client. */ ValkeyModuleCmdFunc reply_callback; /* Reply callback on normal completion.*/ ValkeyModuleAuthCallback auth_reply_cb; /* Reply callback on completing blocking module authentication. */ ValkeyModuleCmdFunc timeout_callback; /* Reply callback on timeout. */ ValkeyModuleDisconnectFunc disconnect_callback; /* Called on disconnection.*/ void (*free_privdata)(ValkeyModuleCtx *, void *); /* privdata cleanup callback.*/ void *privdata; /* Module private data that may be used by the reply or timeout callback. It is set via the ValkeyModule_UnblockClient() API. */ client *thread_safe_ctx_client; /* Fake client to be used for thread safe context so that no lock is required. */ client *reply_client; /* Fake client used to accumulate replies in thread safe contexts. */ int dbid; /* Database number selected by the original client. */ int blocked_on_keys; /* If blocked via VM_BlockClientOnKeys(). */ int unblocked; /* Already on the moduleUnblocked list. */ monotime background_timer; /* Timer tracking the start of background work */ uint64_t background_duration; /* Current command background time duration. Used for measuring latency of blocking cmds */ } ValkeyModuleBlockedClient; /* This is a list of Module Auth Contexts. Each time a Module registers a callback, a new ctx is * added to this list. Multiple modules can register auth callbacks and the same Module can have * multiple auth callbacks. */ static list *moduleAuthCallbacks; static pthread_mutex_t moduleUnblockedClientsMutex = PTHREAD_MUTEX_INITIALIZER; static list *moduleUnblockedClients; /* Pool for temporary client objects. Creating and destroying a client object is * costly. We manage a pool of clients to avoid this cost. Pool expands when * more clients are needed and shrinks when unused. Please see modulesCron() * for more details. */ static client **moduleTempClients; static size_t moduleTempClientCap = 0; static size_t moduleTempClientCount = 0; /* Client count in pool */ static size_t moduleTempClientMinCount = 0; /* Min client count in pool since the last cron. */ /* We need a mutex that is unlocked / relocked in beforeSleep() in order to * allow thread safe contexts to execute commands at a safe moment. */ static pthread_mutex_t moduleGIL = PTHREAD_MUTEX_INITIALIZER; /* Function pointer type for keyspace event notification subscriptions from modules. */ typedef int (*ValkeyModuleNotificationFunc)(ValkeyModuleCtx *ctx, int type, const char *event, ValkeyModuleString *key); /* Function pointer type for post jobs */ typedef void (*ValkeyModulePostNotificationJobFunc)(ValkeyModuleCtx *ctx, void *pd); /* Keyspace notification subscriber information. * See VM_SubscribeToKeyspaceEvents() for more information. */ typedef struct ValkeyModuleKeyspaceSubscriber { /* The module subscribed to the event */ ValkeyModule *module; /* Notification callback in the module*/ ValkeyModuleNotificationFunc notify_callback; /* A bit mask of the events the module is interested in */ int event_mask; /* Active flag set on entry, to avoid reentrant subscribers * calling themselves */ int active; } ValkeyModuleKeyspaceSubscriber; typedef struct ValkeyModulePostExecUnitJob { /* The module subscribed to the event */ ValkeyModule *module; ValkeyModulePostNotificationJobFunc callback; void *pd; void (*free_pd)(void *); int dbid; } ValkeyModulePostExecUnitJob; /* The module keyspace notification subscribers list */ static list *moduleKeyspaceSubscribers; /* The module post keyspace jobs list */ static list *modulePostExecUnitJobs; /* Data structures related to the exported dictionary data structure. */ typedef struct ValkeyModuleDict { rax *rax; /* The radix tree. */ } ValkeyModuleDict; typedef struct ValkeyModuleDictIter { ValkeyModuleDict *dict; raxIterator ri; } ValkeyModuleDictIter; typedef struct ValkeyModuleCommandFilterCtx { ValkeyModuleString **argv; int argv_len; int argc; client *c; } ValkeyModuleCommandFilterCtx; typedef void (*ValkeyModuleCommandFilterFunc)(ValkeyModuleCommandFilterCtx *filter); typedef struct ValkeyModuleCommandFilter { /* The module that registered the filter */ ValkeyModule *module; /* Filter callback function */ ValkeyModuleCommandFilterFunc callback; /* VALKEYMODULE_CMDFILTER_* flags */ int flags; } ValkeyModuleCommandFilter; /* Registered filters */ static list *moduleCommandFilters; typedef void (*ValkeyModuleForkDoneHandler)(int exitcode, int bysignal, void *user_data); static struct ValkeyModuleForkInfo { ValkeyModuleForkDoneHandler done_handler; void *done_handler_user_data; } moduleForkInfo = {0}; typedef struct ValkeyModuleServerInfoData { rax *rax; /* parsed info data. */ } ValkeyModuleServerInfoData; /* Flags for moduleCreateArgvFromUserFormat(). */ #define VALKEYMODULE_ARGV_REPLICATE (1 << 0) #define VALKEYMODULE_ARGV_NO_AOF (1 << 1) #define VALKEYMODULE_ARGV_NO_REPLICAS (1 << 2) #define VALKEYMODULE_ARGV_RESP_3 (1 << 3) #define VALKEYMODULE_ARGV_RESP_AUTO (1 << 4) #define VALKEYMODULE_ARGV_RUN_AS_USER (1 << 5) #define VALKEYMODULE_ARGV_SCRIPT_MODE (1 << 6) #define VALKEYMODULE_ARGV_NO_WRITES (1 << 7) #define VALKEYMODULE_ARGV_CALL_REPLIES_AS_ERRORS (1 << 8) #define VALKEYMODULE_ARGV_RESPECT_DENY_OOM (1 << 9) #define VALKEYMODULE_ARGV_DRY_RUN (1 << 10) #define VALKEYMODULE_ARGV_ALLOW_BLOCK (1 << 11) /* Determine whether the server should signalModifiedKey implicitly. * In case 'ctx' has no 'module' member (and therefore no module->options), * we assume default behavior, that is, the server signals. * (see VM_GetThreadSafeContext) */ #define SHOULD_SIGNAL_MODIFIED_KEYS(ctx) \ ((ctx)->module ? !((ctx)->module->options & VALKEYMODULE_OPTION_NO_IMPLICIT_SIGNAL_MODIFIED) : 1) /* Server events hooks data structures and defines: this modules API * allow modules to subscribe to certain events in the server, such as * the start and end of an RDB or AOF save, the change of role in replication, * and similar other events. */ typedef struct ValkeyModuleEventListener { ValkeyModule *module; ValkeyModuleEvent event; ValkeyModuleEventCallback callback; } ValkeyModuleEventListener; list *ValkeyModule_EventListeners; /* Global list of all the active events. */ /* Data structures related to the module users */ /* This is the object returned by VM_CreateModuleUser(). The module API is * able to create users, set ACLs to such users, and later authenticate * clients using such newly created users. */ typedef struct ValkeyModuleUser { user *user; /* Reference to the real user */ int free_user; /* Indicates that user should also be freed when this object is freed */ } ValkeyModuleUser; /* This is a structure used to export some meta-information such as dbid to the module. */ typedef struct ValkeyModuleKeyOptCtx { struct serverObject *from_key, *to_key; /* Optional name of key processed, NULL when unknown. In most cases, only 'from_key' is valid, but in callbacks such as `copy2`, both 'from_key' and 'to_key' are valid. */ int from_dbid, to_dbid; /* The dbid of the key being processed, -1 when unknown. In most cases, only 'from_dbid' is valid, but in callbacks such as `copy2`, 'from_dbid' and 'to_dbid' are both valid. */ } ValkeyModuleKeyOptCtx; /* Data structures related to module configurations */ /* The function signatures for module config get callbacks. These are identical to the ones exposed in valkeymodule.h. */ typedef ValkeyModuleString *(*ValkeyModuleConfigGetStringFunc)(const char *name, void *privdata); typedef long long (*ValkeyModuleConfigGetNumericFunc)(const char *name, void *privdata); typedef int (*ValkeyModuleConfigGetBoolFunc)(const char *name, void *privdata); typedef int (*ValkeyModuleConfigGetEnumFunc)(const char *name, void *privdata); /* The function signatures for module config set callbacks. These are identical to the ones exposed in valkeymodule.h. */ typedef int (*ValkeyModuleConfigSetStringFunc)(const char *name, ValkeyModuleString *val, void *privdata, ValkeyModuleString **err); typedef int (*ValkeyModuleConfigSetNumericFunc)(const char *name, long long val, void *privdata, ValkeyModuleString **err); typedef int (*ValkeyModuleConfigSetBoolFunc)(const char *name, int val, void *privdata, ValkeyModuleString **err); typedef int (*ValkeyModuleConfigSetEnumFunc)(const char *name, int val, void *privdata, ValkeyModuleString **err); /* Apply signature, identical to valkeymodule.h */ typedef int (*ValkeyModuleConfigApplyFunc)(ValkeyModuleCtx *ctx, void *privdata, ValkeyModuleString **err); /* Struct representing a module config. These are stored in a list in the module struct */ struct ModuleConfig { sds name; /* Name of config without the module name appended to the front */ void *privdata; /* Optional data passed into the module config callbacks */ union get_fn { /* The get callback specified by the module */ ValkeyModuleConfigGetStringFunc get_string; ValkeyModuleConfigGetNumericFunc get_numeric; ValkeyModuleConfigGetBoolFunc get_bool; ValkeyModuleConfigGetEnumFunc get_enum; } get_fn; union set_fn { /* The set callback specified by the module */ ValkeyModuleConfigSetStringFunc set_string; ValkeyModuleConfigSetNumericFunc set_numeric; ValkeyModuleConfigSetBoolFunc set_bool; ValkeyModuleConfigSetEnumFunc set_enum; } set_fn; ValkeyModuleConfigApplyFunc apply_fn; ValkeyModule *module; }; typedef struct ValkeyModuleAsyncRMCallPromise { size_t ref_count; void *private_data; ValkeyModule *module; ValkeyModuleOnUnblocked on_unblocked; client *c; ValkeyModuleCtx *ctx; } ValkeyModuleAsyncRMCallPromise; /* -------------------------------------------------------------------------- * Prototypes * -------------------------------------------------------------------------- */ void VM_FreeCallReply(ValkeyModuleCallReply *reply); void VM_CloseKey(ValkeyModuleKey *key); void autoMemoryCollect(ValkeyModuleCtx *ctx); robj **moduleCreateArgvFromUserFormat(const char *cmdname, const char *fmt, int *argcp, int *flags, va_list ap); void VM_ZsetRangeStop(ValkeyModuleKey *kp); static void zsetKeyReset(ValkeyModuleKey *key); static void moduleInitKeyTypeSpecific(ValkeyModuleKey *key); void VM_FreeDict(ValkeyModuleCtx *ctx, ValkeyModuleDict *d); void VM_FreeServerInfo(ValkeyModuleCtx *ctx, ValkeyModuleServerInfoData *data); /* Helpers for VM_SetCommandInfo. */ static int moduleValidateCommandInfo(const ValkeyModuleCommandInfo *info); static int64_t moduleConvertKeySpecsFlags(int64_t flags, int from_api); static int moduleValidateCommandArgs(ValkeyModuleCommandArg *args, const ValkeyModuleCommandInfoVersion *version); static struct serverCommandArg *moduleCopyCommandArgs(ValkeyModuleCommandArg *args, const ValkeyModuleCommandInfoVersion *version); static serverCommandArgType moduleConvertArgType(ValkeyModuleCommandArgType type, int *error); static int moduleConvertArgFlags(int flags); void moduleCreateContext(ValkeyModuleCtx *out_ctx, ValkeyModule *module, int ctx_flags); /* Common helper functions. */ int moduleVerifyResourceName(const char *name); /* -------------------------------------------------------------------------- * ## Heap allocation raw functions * * Memory allocated with these functions are taken into account by key * eviction algorithms and are reported in memory usage information. * -------------------------------------------------------------------------- */ /* Use like malloc(). Memory allocated with this function is reported in * INFO memory, used for keys eviction according to maxmemory settings * and in general is taken into account as memory allocated by the server. * You should avoid using malloc(). * This function panics if unable to allocate enough memory. */ void *VM_Alloc(size_t bytes) { /* Use 'zmalloc_usable()' instead of 'zmalloc()' to allow the compiler * to recognize the additional memory size, which means that modules can * use the memory reported by 'VM_MallocUsableSize()' safely. In theory this * isn't really needed since this API can't be inlined (not even for embedded * modules like TLS (we use function pointers for module APIs), and the API doesn't * have the malloc_size attribute, but it's hard to predict how smart future compilers * will be, so better safe than sorry. */ return zmalloc_usable(bytes, NULL); } /* Similar to VM_Alloc, but returns NULL in case of allocation failure, instead * of panicking. */ void *VM_TryAlloc(size_t bytes) { return ztrymalloc_usable(bytes, NULL); } /* Use like calloc(). Memory allocated with this function is reported in * INFO memory, used for keys eviction according to maxmemory settings * and in general is taken into account as memory allocated by the server. * You should avoid using calloc() directly. */ void *VM_Calloc(size_t nmemb, size_t size) { return zcalloc_usable(nmemb * size, NULL); } /* Similar to VM_Calloc, but returns NULL in case of allocation failure, instead * of panicking. */ void *VM_TryCalloc(size_t nmemb, size_t size) { return ztrycalloc_usable(nmemb * size, NULL); } /* Use like realloc() for memory obtained with ValkeyModule_Alloc(). */ void *VM_Realloc(void *ptr, size_t bytes) { return zrealloc_usable(ptr, bytes, NULL); } /* Similar to VM_Realloc, but returns NULL in case of allocation failure, * instead of panicking. */ void *VM_TryRealloc(void *ptr, size_t bytes) { return ztryrealloc_usable(ptr, bytes, NULL); } /* Use like free() for memory obtained by ValkeyModule_Alloc() and * ValkeyModule_Realloc(). However you should never try to free with * ValkeyModule_Free() memory allocated with malloc() inside your module. */ void VM_Free(void *ptr) { zfree(ptr); } /* Like strdup() but returns memory allocated with ValkeyModule_Alloc(). */ char *VM_Strdup(const char *str) { return zstrdup(str); } /* -------------------------------------------------------------------------- * Pool allocator * -------------------------------------------------------------------------- */ /* Release the chain of blocks used for pool allocations. */ void poolAllocRelease(ValkeyModuleCtx *ctx) { ValkeyModulePoolAllocBlock *head = ctx->pa_head, *next; while (head != NULL) { next = head->next; zfree(head); head = next; } ctx->pa_head = NULL; } /* Return heap allocated memory that will be freed automatically when the * module callback function returns. Mostly suitable for small allocations * that are short living and must be released when the callback returns * anyway. The returned memory is aligned to the architecture word size * if at least word size bytes are requested, otherwise it is just * aligned to the next power of two, so for example a 3 bytes request is * 4 bytes aligned while a 2 bytes request is 2 bytes aligned. * * There is no realloc style function since when this is needed to use the * pool allocator is not a good idea. * * The function returns NULL if `bytes` is 0. */ void *VM_PoolAlloc(ValkeyModuleCtx *ctx, size_t bytes) { if (bytes == 0) return NULL; ValkeyModulePoolAllocBlock *b = ctx->pa_head; size_t left = b ? b->size - b->used : 0; /* Fix alignment. */ if (left >= bytes) { size_t alignment = VALKEYMODULE_POOL_ALLOC_ALIGN; while (bytes < alignment && alignment / 2 >= bytes) alignment /= 2; if (b->used % alignment) b->used += alignment - (b->used % alignment); left = (b->used > b->size) ? 0 : b->size - b->used; } /* Create a new block if needed. */ if (left < bytes) { size_t blocksize = VALKEYMODULE_POOL_ALLOC_MIN_SIZE; if (blocksize < bytes) blocksize = bytes; b = zmalloc(sizeof(*b) + blocksize); b->size = blocksize; b->used = 0; b->next = ctx->pa_head; ctx->pa_head = b; } char *retval = b->memory + b->used; b->used += bytes; return retval; } /* -------------------------------------------------------------------------- * Helpers for modules API implementation * -------------------------------------------------------------------------- */ static void initClientModuleData(client *c) { if (c->module_data) return; c->module_data = zcalloc(sizeof(ClientModuleData)); } void freeClientModuleData(client *c) { if (!c->module_data) return; /* Free the ValkeyModuleBlockedClient held onto for reprocessing if not already freed. */ zfree(c->module_data->module_blocked_client); zfree(c->module_data); c->module_data = NULL; } void moduleEnqueueLoadModule(sds path, sds *argv, int argc) { int i; struct moduleLoadQueueEntry *loadmod; loadmod = zmalloc(sizeof(struct moduleLoadQueueEntry)); loadmod->argv = argc ? zmalloc(sizeof(robj *) * argc) : NULL; loadmod->path = sdsnew(path); loadmod->argc = argc; for (i = 0; i < argc; i++) { loadmod->argv[i] = createRawStringObject(argv[i], sdslen(argv[i])); } listAddNodeTail(server.loadmodule_queue, loadmod); } sds moduleLoadQueueEntryToLoadmoduleOptionStr(ValkeyModule *module, const char *config_option_str) { sds line; line = sdsnew(config_option_str); line = sdscatlen(line, " ", 1); line = sdscatsds(line, module->loadmod->path); for (int i = 0; i < module->loadmod->argc; i++) { line = sdscatlen(line, " ", 1); line = sdscatsds(line, module->loadmod->argv[i]->ptr); } return line; } client *moduleAllocTempClient(void) { client *c = NULL; if (moduleTempClientCount > 0) { c = moduleTempClients[--moduleTempClientCount]; if (moduleTempClientCount < moduleTempClientMinCount) moduleTempClientMinCount = moduleTempClientCount; } else { c = createClient(NULL); c->flag.module = 1; c->flag.fake = 1; c->user = NULL; /* Root user */ } return c; } static void freeValkeyModuleAsyncRMCallPromise(ValkeyModuleAsyncRMCallPromise *promise) { if (--promise->ref_count > 0) { return; } /* When the promise is finally freed it can not have a client attached to it. * Either releasing the client or VM_CallReplyPromiseAbort would have removed it. */ serverAssert(!promise->c); zfree(promise); } void moduleReleaseTempClient(client *c) { if (moduleTempClientCount == moduleTempClientCap) { moduleTempClientCap = moduleTempClientCap ? moduleTempClientCap * 2 : 32; moduleTempClients = zrealloc(moduleTempClients, sizeof(c) * moduleTempClientCap); } clearClientConnectionState(c); listEmpty(c->reply); c->reply_bytes = 0; c->duration = 0; resetClient(c); c->bufpos = 0; c->raw_flag = 0; c->flag.module = 1; c->flag.fake = 1; c->user = NULL; /* Root user */ c->cmd = c->lastcmd = c->realcmd = c->io_parsed_cmd = NULL; if (c->bstate && c->bstate->async_rm_call_handle) { ValkeyModuleAsyncRMCallPromise *promise = c->bstate->async_rm_call_handle; promise->c = NULL; /* Remove the client from the promise so it will no longer be possible to abort it. */ freeValkeyModuleAsyncRMCallPromise(promise); c->bstate->async_rm_call_handle = NULL; } moduleTempClients[moduleTempClientCount++] = c; } /* Create an empty key of the specified type. `key` must point to a key object * opened for writing where the `.value` member is set to NULL because the * key was found to be non existing. * * On success VALKEYMODULE_OK is returned and the key is populated with * the value of the specified type. The function fails and returns * VALKEYMODULE_ERR if: * * 1. The key is not open for writing. * 2. The key is not empty. * 3. The specified type is unknown. */ int moduleCreateEmptyKey(ValkeyModuleKey *key, int type) { robj *obj; /* The key must be open for writing and non existing to proceed. */ if (!(key->mode & VALKEYMODULE_WRITE) || key->value) return VALKEYMODULE_ERR; switch (type) { case VALKEYMODULE_KEYTYPE_LIST: obj = createListListpackObject(); break; case VALKEYMODULE_KEYTYPE_ZSET: obj = createZsetListpackObject(); break; case VALKEYMODULE_KEYTYPE_HASH: obj = createHashObject(); break; case VALKEYMODULE_KEYTYPE_STREAM: obj = createStreamObject(); break; default: return VALKEYMODULE_ERR; } dbAdd(key->db, key->key, &obj); key->value = obj; moduleInitKeyTypeSpecific(key); return VALKEYMODULE_OK; } /* Frees key->iter and sets it to NULL. */ static void moduleFreeKeyIterator(ValkeyModuleKey *key) { serverAssert(key->iter != NULL); switch (key->value->type) { case OBJ_LIST: listTypeReleaseIterator(key->iter); break; case OBJ_STREAM: streamIteratorStop(key->iter); zfree(key->iter); break; default: serverAssert(0); /* No key->iter for other types. */ } key->iter = NULL; } /* Callback for listTypeTryConversion(). * Frees list iterator and sets it to NULL. */ static void moduleFreeListIterator(void *data) { ValkeyModuleKey *key = (ValkeyModuleKey *)data; serverAssert(key->value->type == OBJ_LIST); if (key->iter) moduleFreeKeyIterator(key); } /* This function is called in low-level API implementation functions in order * to check if the value associated with the key remained empty after an * operation that removed elements from an aggregate data type. * * If this happens, the key is deleted from the DB and the key object state * is set to the right one in order to be targeted again by write operations * possibly recreating the key if needed. * * The function returns 1 if the key value object is found empty and is * deleted, otherwise 0 is returned. */ int moduleDelKeyIfEmpty(ValkeyModuleKey *key) { if (!(key->mode & VALKEYMODULE_WRITE) || key->value == NULL) return 0; int isempty; robj *o = key->value; switch (o->type) { case OBJ_LIST: isempty = listTypeLength(o) == 0; break; case OBJ_SET: isempty = setTypeSize(o) == 0; break; case OBJ_ZSET: isempty = zsetLength(o) == 0; break; case OBJ_HASH: isempty = hashTypeLength(o) == 0; break; case OBJ_STREAM: isempty = streamLength(o) == 0; break; default: isempty = 0; } if (isempty) { if (key->iter) moduleFreeKeyIterator(key); dbDelete(key->db, key->key); key->value = NULL; return 1; } else { return 0; } } /* -------------------------------------------------------------------------- * Service API exported to modules * * Note that all the exported APIs are called VM_ in the core * and ValkeyModule_ in the module side (defined as function * pointers in valkeymodule.h). In this way the dynamic linker does not * mess with our global function pointers, overriding it with the symbols * defined in the main executable having the same names. * -------------------------------------------------------------------------- */ int VM_GetApi(const char *funcname, void **targetPtrPtr) { /* Lookup the requested module API and store the function pointer into the * target pointer. The function returns VALKEYMODULE_ERR if there is no such * named API, otherwise VALKEYMODULE_OK. * * This function is not meant to be used by modules developer, it is only * used implicitly by including valkeymodule.h. */ dictEntry *he = dictFind(server.moduleapi, funcname); if (!he) return VALKEYMODULE_ERR; *targetPtrPtr = dictGetVal(he); return VALKEYMODULE_OK; } void modulePostExecutionUnitOperations(void) { if (server.execution_nesting) return; if (server.busy_module_yield_flags) { blockingOperationEnds(); server.busy_module_yield_flags = BUSY_MODULE_YIELD_NONE; if (server.current_client) unprotectClient(server.current_client); unblockPostponedClients(); } } /* Free the context after the user function was called. */ void moduleFreeContext(ValkeyModuleCtx *ctx) { /* See comment in moduleCreateContext */ if (!(ctx->flags & (VALKEYMODULE_CTX_THREAD_SAFE | VALKEYMODULE_CTX_COMMAND))) { exitExecutionUnit(); postExecutionUnitOperations(); } autoMemoryCollect(ctx); poolAllocRelease(ctx); if (ctx->postponed_arrays) { zfree(ctx->postponed_arrays); ctx->postponed_arrays_count = 0; serverLog(LL_WARNING, "API misuse detected in module %s: " "ValkeyModule_ReplyWith*(VALKEYMODULE_POSTPONED_LEN) " "not matched by the same number of ValkeyModule_SetReply*Len() " "calls.", ctx->module->name); } /* If this context has a temp client, we return it back to the pool. * If this context created a new client (e.g detached context), we free it. * If the client is assigned manually, e.g ctx->client = someClientInstance, * none of these flags will be set and we do not attempt to free it. */ if (ctx->flags & VALKEYMODULE_CTX_TEMP_CLIENT) moduleReleaseTempClient(ctx->client); else if (ctx->flags & VALKEYMODULE_CTX_NEW_CLIENT) freeClient(ctx->client); } static CallReply *moduleParseReply(client *c, ValkeyModuleCtx *ctx) { /* Convert the result of the command into a module reply. */ sds proto = sdsnewlen(c->buf, c->bufpos); c->bufpos = 0; while (listLength(c->reply)) { clientReplyBlock *o = listNodeValue(listFirst(c->reply)); proto = sdscatlen(proto, o->buf, o->used); listDelNode(c->reply, listFirst(c->reply)); } CallReply *reply = callReplyCreate(proto, c->deferred_reply_errors, ctx); c->deferred_reply_errors = NULL; /* now the responsibility of the reply object. */ return reply; } void moduleCallCommandUnblockedHandler(client *c) { ValkeyModuleCtx ctx; ValkeyModuleAsyncRMCallPromise *promise = c->bstate->async_rm_call_handle; serverAssert(promise); ValkeyModule *module = promise->module; if (!promise->on_unblocked) { moduleReleaseTempClient(c); return; /* module did not set any unblock callback. */ } moduleCreateContext(&ctx, module, VALKEYMODULE_CTX_TEMP_CLIENT); selectDb(ctx.client, c->db->id); CallReply *reply = moduleParseReply(c, NULL); module->in_call++; promise->on_unblocked(&ctx, reply, promise->private_data); module->in_call--; moduleFreeContext(&ctx); moduleReleaseTempClient(c); } /* Allocates the memory necessary to hold the ValkeyModuleCtx structure, and * returns the pointer to the allocated memory. * * Used by the scripting engines implementation to cache the context structure. */ ValkeyModuleCtx *moduleAllocateContext(void) { return (ValkeyModuleCtx *)zcalloc(sizeof(ValkeyModuleCtx)); } /* Create a module ctx and keep track of the nesting level. * * Note: When creating ctx for threads (VM_GetThreadSafeContext and * VM_GetDetachedThreadSafeContext) we do not bump up the nesting level * because we only need to track of nesting level in the main thread * (only the main thread uses propagatePendingCommands) */ void moduleCreateContext(ValkeyModuleCtx *out_ctx, ValkeyModule *module, int ctx_flags) { memset(out_ctx, 0, sizeof(ValkeyModuleCtx)); out_ctx->getapifuncptr = (void *)(unsigned long)&VM_GetApi; out_ctx->module = module; out_ctx->flags = ctx_flags; if (ctx_flags & VALKEYMODULE_CTX_TEMP_CLIENT) out_ctx->client = moduleAllocTempClient(); else if (ctx_flags & VALKEYMODULE_CTX_NEW_CLIENT) { out_ctx->client = createClient(NULL); out_ctx->client->flag.fake = 1; } /* Calculate the initial yield time for long blocked contexts. * in loading we depend on the server hz, but in other cases we also wait * for busy_reply_threshold. * Note that in theory we could have started processing BUSY_MODULE_YIELD_EVENTS * sooner, and only delay the processing for clients till the busy_reply_threshold, * but this carries some overheads of frequently marking clients with BLOCKED_POSTPONE * and releasing them, i.e. if modules only block for short periods. */ if (server.loading) out_ctx->next_yield_time = getMonotonicUs() + 1000000 / server.hz; else out_ctx->next_yield_time = getMonotonicUs() + server.busy_reply_threshold * 1000; /* Increment the execution_nesting counter (module is about to execute some code), * except in the following cases: * 1. We came here from cmd->proc (either call() or AOF load). * In the former, the counter has been already incremented from within * call() and in the latter we don't care about execution_nesting * 2. If we are running in a thread (execution_nesting will be dealt with * when locking/unlocking the GIL) */ if (!(ctx_flags & (VALKEYMODULE_CTX_THREAD_SAFE | VALKEYMODULE_CTX_COMMAND))) { enterExecutionUnit(1, 0); } } /* Initialize a module context to be used by scripting engines callback * functions. */ void moduleScriptingEngineInitContext(ValkeyModuleCtx *out_ctx, ValkeyModule *module, client *client) { moduleCreateContext(out_ctx, module, VALKEYMODULE_CTX_NONE); out_ctx->client = client; } /* This command binds the normal command invocation with commands * exported by modules. */ void ValkeyModuleCommandDispatcher(client *c) { ValkeyModuleCommand *cp = c->cmd->module_cmd; ValkeyModuleCtx ctx; moduleCreateContext(&ctx, cp->module, VALKEYMODULE_CTX_COMMAND); ctx.client = c; cp->func(&ctx, (void **)c->argv, c->argc); moduleFreeContext(&ctx); /* In some cases processMultibulkBuffer uses sdsMakeRoomFor to * expand the query buffer, and in order to avoid a big object copy * the query buffer SDS may be used directly as the SDS string backing * the client argument vectors: sometimes this will result in the SDS * string having unused space at the end. Later if a module takes ownership * of the RedisString, such space will be wasted forever. Inside the * server core this is not a problem because tryObjectEncoding() is called * before storing strings in the key space. Here we need to do it * for the module. */ for (int i = 0; i < c->argc; i++) { /* Only do the work if the module took ownership of the object: * in that case the refcount is no longer 1. */ if (c->argv[i]->refcount > 1) trimStringObjectIfNeeded(c->argv[i], 0); } } /* This function returns the list of keys, with the same interface as the * 'getkeys' function of the native commands, for module commands that exported * the "getkeys-api" flag during the registration. This is done when the * list of keys are not at fixed positions, so that first/last/step cannot * be used. * * In order to accomplish its work, the module command is called, flagging * the context in a way that the command can recognize this is a special * "get keys" call by calling ValkeyModule_IsKeysPositionRequest(ctx). */ int moduleGetCommandKeysViaAPI(struct serverCommand *cmd, robj **argv, int argc, getKeysResult *result) { ValkeyModuleCommand *cp = cmd->module_cmd; ValkeyModuleCtx ctx; moduleCreateContext(&ctx, cp->module, VALKEYMODULE_CTX_KEYS_POS_REQUEST); /* Initialize getKeysResult */ getKeysPrepareResult(result, MAX_KEYS_BUFFER); ctx.keys_result = result; cp->func(&ctx, (void **)argv, argc); /* We currently always use the array allocated by VM_KeyAtPos() and don't try * to optimize for the pre-allocated buffer. */ moduleFreeContext(&ctx); return result->numkeys; } /* This function returns the list of channels, with the same interface as * moduleGetCommandKeysViaAPI, for modules that declare "getchannels-api" * during registration. Unlike keys, this is the only way to declare channels. */ int moduleGetCommandChannelsViaAPI(struct serverCommand *cmd, robj **argv, int argc, getKeysResult *result) { ValkeyModuleCommand *cp = cmd->module_cmd; ValkeyModuleCtx ctx; moduleCreateContext(&ctx, cp->module, VALKEYMODULE_CTX_CHANNELS_POS_REQUEST); /* Initialize getKeysResult */ getKeysPrepareResult(result, MAX_KEYS_BUFFER); ctx.keys_result = result; cp->func(&ctx, (void **)argv, argc); /* We currently always use the array allocated by VM_RM_ChannelAtPosWithFlags() and don't try * to optimize for the pre-allocated buffer. */ moduleFreeContext(&ctx); return result->numkeys; } /* -------------------------------------------------------------------------- * ## Commands API * * These functions are used to implement custom commands. * * For examples, see https://valkey.io/topics/modules-intro. * -------------------------------------------------------------------------- */ /* Return non-zero if a module command, that was declared with the * flag "getkeys-api", is called in a special way to get the keys positions * and not to get executed. Otherwise zero is returned. */ int VM_IsKeysPositionRequest(ValkeyModuleCtx *ctx) { return (ctx->flags & VALKEYMODULE_CTX_KEYS_POS_REQUEST) != 0; } /* When a module command is called in order to obtain the position of * keys, since it was flagged as "getkeys-api" during the registration, * the command implementation checks for this special call using the * ValkeyModule_IsKeysPositionRequest() API and uses this function in * order to report keys. * * The supported flags are the ones used by VM_SetCommandInfo, see VALKEYMODULE_CMD_KEY_*. * * * The following is an example of how it could be used: * * if (ValkeyModule_IsKeysPositionRequest(ctx)) { * ValkeyModule_KeyAtPosWithFlags(ctx, 2, VALKEYMODULE_CMD_KEY_RO | VALKEYMODULE_CMD_KEY_ACCESS); * ValkeyModule_KeyAtPosWithFlags(ctx, 1, VALKEYMODULE_CMD_KEY_RW | VALKEYMODULE_CMD_KEY_UPDATE | * VALKEYMODULE_CMD_KEY_ACCESS); * } * * Note: in the example above the get keys API could have been handled by key-specs (preferred). * Implementing the getkeys-api is required only when is it not possible to declare key-specs that cover all keys. * */ void VM_KeyAtPosWithFlags(ValkeyModuleCtx *ctx, int pos, int flags) { if (!(ctx->flags & VALKEYMODULE_CTX_KEYS_POS_REQUEST) || !ctx->keys_result) return; if (pos <= 0) return; getKeysResult *res = ctx->keys_result; /* Check overflow */ if (res->numkeys == res->size) { int newsize = res->size + (res->size > 8192 ? 8192 : res->size); getKeysPrepareResult(res, newsize); } res->keys[res->numkeys].pos = pos; res->keys[res->numkeys].flags = moduleConvertKeySpecsFlags(flags, 1); res->numkeys++; } /* This API existed before VM_KeyAtPosWithFlags was added, now deprecated and * can be used for compatibility with older versions, before key-specs and flags * were introduced. */ void VM_KeyAtPos(ValkeyModuleCtx *ctx, int pos) { /* Default flags require full access */ int flags = moduleConvertKeySpecsFlags(CMD_KEY_FULL_ACCESS, 0); VM_KeyAtPosWithFlags(ctx, pos, flags); } /* Return non-zero if a module command, that was declared with the * flag "getchannels-api", is called in a special way to get the channel positions * and not to get executed. Otherwise zero is returned. */ int VM_IsChannelsPositionRequest(ValkeyModuleCtx *ctx) { return (ctx->flags & VALKEYMODULE_CTX_CHANNELS_POS_REQUEST) != 0; } /* When a module command is called in order to obtain the position of * channels, since it was flagged as "getchannels-api" during the * registration, the command implementation checks for this special call * using the ValkeyModule_IsChannelsPositionRequest() API and uses this * function in order to report the channels. * * The supported flags are: * * VALKEYMODULE_CMD_CHANNEL_SUBSCRIBE: This command will subscribe to the channel. * * VALKEYMODULE_CMD_CHANNEL_UNSUBSCRIBE: This command will unsubscribe from this channel. * * VALKEYMODULE_CMD_CHANNEL_PUBLISH: This command will publish to this channel. * * VALKEYMODULE_CMD_CHANNEL_PATTERN: Instead of acting on a specific channel, will act on any * channel specified by the pattern. This is the same access * used by the PSUBSCRIBE and PUNSUBSCRIBE commands. * Not intended to be used with PUBLISH permissions. * * The following is an example of how it could be used: * * if (ValkeyModule_IsChannelsPositionRequest(ctx)) { * ValkeyModule_ChannelAtPosWithFlags(ctx, 1, VALKEYMODULE_CMD_CHANNEL_SUBSCRIBE | * VALKEYMODULE_CMD_CHANNEL_PATTERN); ValkeyModule_ChannelAtPosWithFlags(ctx, 1, VALKEYMODULE_CMD_CHANNEL_PUBLISH); * } * * Note: One usage of declaring channels is for evaluating ACL permissions. In this context, * unsubscribing is always allowed, so commands will only be checked against subscribe and * publish permissions. This is preferred over using VM_ACLCheckChannelPermissions, since * it allows the ACLs to be checked before the command is executed. */ void VM_ChannelAtPosWithFlags(ValkeyModuleCtx *ctx, int pos, int flags) { if (!(ctx->flags & VALKEYMODULE_CTX_CHANNELS_POS_REQUEST) || !ctx->keys_result) return; if (pos <= 0) return; getKeysResult *res = ctx->keys_result; /* Check overflow */ if (res->numkeys == res->size) { int newsize = res->size + (res->size > 8192 ? 8192 : res->size); getKeysPrepareResult(res, newsize); } int new_flags = 0; if (flags & VALKEYMODULE_CMD_CHANNEL_SUBSCRIBE) new_flags |= CMD_CHANNEL_SUBSCRIBE; if (flags & VALKEYMODULE_CMD_CHANNEL_UNSUBSCRIBE) new_flags |= CMD_CHANNEL_UNSUBSCRIBE; if (flags & VALKEYMODULE_CMD_CHANNEL_PUBLISH) new_flags |= CMD_CHANNEL_PUBLISH; if (flags & VALKEYMODULE_CMD_CHANNEL_PATTERN) new_flags |= CMD_CHANNEL_PATTERN; res->keys[res->numkeys].pos = pos; res->keys[res->numkeys].flags = new_flags; res->numkeys++; } /* Returns 1 if name is valid, otherwise returns 0. * * We want to block some chars in module command names that we know can * mess things up. * * There are these characters: * ' ' (space) - issues with old inline protocol. * '\r', '\n' (newline) - can mess up the protocol on acl error replies. * '|' - sub-commands. * '@' - ACL categories. * '=', ',' - info and client list fields (':' handled by getSafeInfoString). * */ int isCommandNameValid(const char *name) { const char *block_chars = " \r\n|@=,"; if (strpbrk(name, block_chars)) return 0; return 1; } /* Helper for VM_CreateCommand(). Turns a string representing command * flags into the command flags used by the server core. * * It returns the set of flags, or -1 if unknown flags are found. */ int64_t commandFlagsFromString(char *s) { int count, j; int64_t flags = 0; sds *tokens = sdssplitlen(s, strlen(s), " ", 1, &count); for (j = 0; j < count; j++) { char *t = tokens[j]; /* clang-format off */ if (!strcasecmp(t,"write")) flags |= CMD_WRITE; else if (!strcasecmp(t,"readonly")) flags |= CMD_READONLY; else if (!strcasecmp(t,"admin")) flags |= CMD_ADMIN; else if (!strcasecmp(t,"deny-oom")) flags |= CMD_DENYOOM; else if (!strcasecmp(t,"deny-script")) flags |= CMD_NOSCRIPT; else if (!strcasecmp(t,"allow-loading")) flags |= CMD_LOADING; else if (!strcasecmp(t,"pubsub")) flags |= CMD_PUBSUB; else if (!strcasecmp(t,"random")) { /* Deprecated. Silently ignore. */ } else if (!strcasecmp(t,"blocking")) flags |= CMD_BLOCKING; else if (!strcasecmp(t,"allow-stale")) flags |= CMD_STALE; else if (!strcasecmp(t,"no-monitor")) flags |= CMD_SKIP_MONITOR; else if (!strcasecmp(t,"no-slowlog")) flags |= CMD_SKIP_COMMANDLOG; else if (!strcasecmp(t,"no-commandlog")) flags |= CMD_SKIP_COMMANDLOG; else if (!strcasecmp(t,"fast")) flags |= CMD_FAST; else if (!strcasecmp(t,"no-auth")) flags |= CMD_NO_AUTH; else if (!strcasecmp(t,"may-replicate")) flags |= CMD_MAY_REPLICATE; else if (!strcasecmp(t,"getkeys-api")) flags |= CMD_MODULE_GETKEYS; else if (!strcasecmp(t,"getchannels-api")) flags |= CMD_MODULE_GETCHANNELS; else if (!strcasecmp(t,"no-cluster")) flags |= CMD_MODULE_NO_CLUSTER; else if (!strcasecmp(t,"no-mandatory-keys")) flags |= CMD_NO_MANDATORY_KEYS; else if (!strcasecmp(t,"allow-busy")) flags |= CMD_ALLOW_BUSY; else break; /* clang-format on */ } sdsfreesplitres(tokens, count); if (j != count) return -1; /* Some token not processed correctly. */ return flags; } ValkeyModuleCommand *moduleCreateCommandProxy(struct ValkeyModule *module, sds declared_name, sds fullname, ValkeyModuleCmdFunc cmdfunc, int64_t flags, int firstkey, int lastkey, int keystep); /* Register a new command in the server, that will be handled by * calling the function pointer 'cmdfunc' using the ValkeyModule calling * convention. * * The function returns VALKEYMODULE_ERR in these cases: * - If creation of module command is called outside the ValkeyModule_OnLoad. * - The specified command is already busy. * - The command name contains some chars that are not allowed. * - A set of invalid flags were passed. * * Otherwise VALKEYMODULE_OK is returned and the new command is registered. * * This function must be called during the initialization of the module * inside the ValkeyModule_OnLoad() function. Calling this function outside * of the initialization function is not defined. * * The command function type is the following: * * int MyCommand_ValkeyCommand(ValkeyModuleCtx *ctx, ValkeyModuleString **argv, int argc); * * And is supposed to always return VALKEYMODULE_OK. * * The set of flags 'strflags' specify the behavior of the command, and should * be passed as a C string composed of space separated words, like for * example "write deny-oom". The set of flags are: * * * **"write"**: The command may modify the data set (it may also read * from it). * * **"readonly"**: The command returns data from keys but never writes. * * **"admin"**: The command is an administrative command (may change * replication or perform similar tasks). * * **"deny-oom"**: The command may use additional memory and should be * denied during out of memory conditions. * * **"deny-script"**: Don't allow this command in Lua scripts. * * **"allow-loading"**: Allow this command while the server is loading data. * Only commands not interacting with the data set * should be allowed to run in this mode. If not sure * don't use this flag. * * **"pubsub"**: The command publishes things on Pub/Sub channels. * * **"random"**: The command may have different outputs even starting * from the same input arguments and key values. * Starting from Redis OSS 7.0 this flag has been deprecated. * Declaring a command as "random" can be done using * command tips, see https://valkey.io/topics/command-tips. * * **"allow-stale"**: The command is allowed to run on replicas that don't * serve stale data. Don't use if you don't know what * this means. * * **"no-monitor"**: Don't propagate the command on monitor. Use this if * the command has sensitive data among the arguments. * * **"no-slowlog"**: Deprecated, please use "no-commandlog". * * **"no-commandlog"**: Don't log this command in the commandlog. Use this if * the command has sensitive data among the arguments. * * **"fast"**: The command time complexity is not greater * than O(log(N)) where N is the size of the collection or * anything else representing the normal scalability * issue with the command. * * **"getkeys-api"**: The command implements the interface to return * the arguments that are keys. Used when start/stop/step * is not enough because of the command syntax. * * **"no-cluster"**: The command should not register in Cluster * since is not designed to work with it because, for * example, is unable to report the position of the * keys, programmatically creates key names, or any * other reason. * * **"no-auth"**: This command can be run by an un-authenticated client. * Normally this is used by a command that is used * to authenticate a client. * * **"may-replicate"**: This command may generate replication traffic, even * though it's not a write command. * * **"no-mandatory-keys"**: All the keys this command may take are optional * * **"blocking"**: The command has the potential to block the client. * * **"allow-busy"**: Permit the command while the server is blocked either by * a script or by a slow module command, see * VM_Yield. * * **"getchannels-api"**: The command implements the interface to return * the arguments that are channels. * * The last three parameters specify which arguments of the new command are * keys. See https://valkey.io/commands/command for more information. * * * `firstkey`: One-based index of the first argument that's a key. * Position 0 is always the command name itself. * 0 for commands with no keys. * * `lastkey`: One-based index of the last argument that's a key. * Negative numbers refer to counting backwards from the last * argument (-1 means the last argument provided) * 0 for commands with no keys. * * `keystep`: Step between first and last key indexes. * 0 for commands with no keys. * * This information is used by ACL, Cluster and the `COMMAND` command. * * NOTE: The scheme described above serves a limited purpose and can * only be used to find keys that exist at constant indices. * For non-trivial key arguments, you may pass 0,0,0 and use * ValkeyModule_SetCommandInfo to set key specs using a more advanced scheme and use * ValkeyModule_SetCommandACLCategories to set ACL categories of the commands. */ int VM_CreateCommand(ValkeyModuleCtx *ctx, const char *name, ValkeyModuleCmdFunc cmdfunc, const char *strflags, int firstkey, int lastkey, int keystep) { if (!ctx->module->onload) return VALKEYMODULE_ERR; int64_t flags = strflags ? commandFlagsFromString((char *)strflags) : 0; if (flags == -1) return VALKEYMODULE_ERR; if ((flags & CMD_MODULE_NO_CLUSTER) && server.cluster_enabled) return VALKEYMODULE_ERR; /* Check if the command name is valid. */ if (!isCommandNameValid(name)) return VALKEYMODULE_ERR; /* Check if the command name is busy. */ if (lookupCommandByCString(name) != NULL) return VALKEYMODULE_ERR; sds declared_name = sdsnew(name); ValkeyModuleCommand *cp = moduleCreateCommandProxy(ctx->module, declared_name, sdsdup(declared_name), cmdfunc, flags, firstkey, lastkey, keystep); cp->serverCmd->arity = cmdfunc ? -1 : -2; /* Default value, can be changed later via dedicated API */ /* Drain IO queue before modifying commands dictionary to prevent concurrent access while modifying it. */ drainIOThreadsQueue(); serverAssert(hashtableAdd(server.commands, cp->serverCmd)); serverAssert(hashtableAdd(server.orig_commands, cp->serverCmd)); cp->serverCmd->id = ACLGetCommandID(declared_name); /* ID used for ACL. */ return VALKEYMODULE_OK; } /* A proxy that help create a module command / subcommand. * * 'declared_name': it contains the sub_name, which is just the fullname for non-subcommands. * 'fullname': sds string representing the command fullname. * * Function will take the ownership of both 'declared_name' and 'fullname' SDS. */ ValkeyModuleCommand *moduleCreateCommandProxy(struct ValkeyModule *module, sds declared_name, sds fullname, ValkeyModuleCmdFunc cmdfunc, int64_t flags, int firstkey, int lastkey, int keystep) { struct serverCommand *serverCmd; ValkeyModuleCommand *cp; /* Create a command "proxy", which is a structure that is referenced * in the command table, so that the generic command that works as * binding between modules and the server, can know what function to call * and what the module is. */ cp = zcalloc(sizeof(*cp)); cp->module = module; cp->func = cmdfunc; cp->serverCmd = zcalloc(sizeof(*serverCmd)); cp->serverCmd->declared_name = declared_name; /* SDS for module commands */ cp->serverCmd->fullname = fullname; cp->serverCmd->group = COMMAND_GROUP_MODULE; cp->serverCmd->proc = ValkeyModuleCommandDispatcher; cp->serverCmd->flags = flags | CMD_MODULE; cp->serverCmd->module_cmd = cp; if (firstkey != 0) { cp->serverCmd->key_specs_num = 1; cp->serverCmd->key_specs = zcalloc(sizeof(keySpec)); cp->serverCmd->key_specs[0].flags = CMD_KEY_FULL_ACCESS; if (flags & CMD_MODULE_GETKEYS) cp->serverCmd->key_specs[0].flags |= CMD_KEY_VARIABLE_FLAGS; cp->serverCmd->key_specs[0].begin_search_type = KSPEC_BS_INDEX; cp->serverCmd->key_specs[0].bs.index.pos = firstkey; cp->serverCmd->key_specs[0].find_keys_type = KSPEC_FK_RANGE; cp->serverCmd->key_specs[0].fk.range.lastkey = lastkey < 0 ? lastkey : (lastkey - firstkey); cp->serverCmd->key_specs[0].fk.range.keystep = keystep; cp->serverCmd->key_specs[0].fk.range.limit = 0; } else { cp->serverCmd->key_specs_num = 0; cp->serverCmd->key_specs = NULL; } populateCommandLegacyRangeSpec(cp->serverCmd); cp->serverCmd->microseconds = 0; cp->serverCmd->calls = 0; cp->serverCmd->rejected_calls = 0; cp->serverCmd->failed_calls = 0; return cp; } /* Get an opaque structure, representing a module command, by command name. * This structure is used in some of the command-related APIs. * * NULL is returned in case of the following errors: * * * Command not found * * The command is not a module command * * The command doesn't belong to the calling module */ ValkeyModuleCommand *VM_GetCommand(ValkeyModuleCtx *ctx, const char *name) { struct serverCommand *cmd = lookupCommandByCString(name); if (!cmd || !(cmd->flags & CMD_MODULE)) return NULL; ValkeyModuleCommand *cp = cmd->module_cmd; if (cp->module != ctx->module) return NULL; return cp; } /* Very similar to ValkeyModule_CreateCommand except that it is used to create * a subcommand, associated with another, container, command. * * Example: If a module has a configuration command, MODULE.CONFIG, then * GET and SET should be individual subcommands, while MODULE.CONFIG is * a command, but should not be registered with a valid `funcptr`: * * if (ValkeyModule_CreateCommand(ctx,"module.config",NULL,"",0,0,0) == VALKEYMODULE_ERR) * return VALKEYMODULE_ERR; * * ValkeyModuleCommand *parent = ValkeyModule_GetCommand(ctx,,"module.config"); * * if (ValkeyModule_CreateSubcommand(parent,"set",cmd_config_set,"",0,0,0) == VALKEYMODULE_ERR) * return VALKEYMODULE_ERR; * * if (ValkeyModule_CreateSubcommand(parent,"get",cmd_config_get,"",0,0,0) == VALKEYMODULE_ERR) * return VALKEYMODULE_ERR; * * Returns VALKEYMODULE_OK on success and VALKEYMODULE_ERR in case of the following errors: * * * Error while parsing `strflags` * * Command is marked as `no-cluster` but cluster mode is enabled * * `parent` is already a subcommand (we do not allow more than one level of command nesting) * * `parent` is a command with an implementation (ValkeyModuleCmdFunc) (A parent command should be a pure container of * subcommands) * * `parent` already has a subcommand called `name` * * Creating a subcommand is called outside of ValkeyModule_OnLoad. */ int VM_CreateSubcommand(ValkeyModuleCommand *parent, const char *name, ValkeyModuleCmdFunc cmdfunc, const char *strflags, int firstkey, int lastkey, int keystep) { if (!parent->module->onload) return VALKEYMODULE_ERR; int64_t flags = strflags ? commandFlagsFromString((char *)strflags) : 0; if (flags == -1) return VALKEYMODULE_ERR; if ((flags & CMD_MODULE_NO_CLUSTER) && server.cluster_enabled) return VALKEYMODULE_ERR; struct serverCommand *parent_cmd = parent->serverCmd; if (parent_cmd->parent) return VALKEYMODULE_ERR; /* We don't allow more than one level of subcommands */ ValkeyModuleCommand *parent_cp = parent_cmd->module_cmd; if (parent_cp->func) return VALKEYMODULE_ERR; /* A parent command should be a pure container of subcommands */ /* Check if the command name is valid. */ if (!isCommandNameValid(name)) return VALKEYMODULE_ERR; /* Check if the command name is busy within the parent command. */ sds declared_name = sdsnew(name); if (parent_cmd->subcommands_ht && lookupSubcommand(parent_cmd, declared_name) != NULL) { sdsfree(declared_name); return VALKEYMODULE_ERR; } sds fullname = catSubCommandFullname(parent_cmd->fullname, name); ValkeyModuleCommand *cp = moduleCreateCommandProxy(parent->module, declared_name, fullname, cmdfunc, flags, firstkey, lastkey, keystep); cp->serverCmd->arity = -2; commandAddSubcommand(parent_cmd, cp->serverCmd); return VALKEYMODULE_OK; } /* Accessors of array elements of structs where the element size is stored * separately in the version struct. */ static ValkeyModuleCommandHistoryEntry *moduleCmdHistoryEntryAt(const ValkeyModuleCommandInfoVersion *version, ValkeyModuleCommandHistoryEntry *entries, int index) { off_t offset = index * version->sizeof_historyentry; return (ValkeyModuleCommandHistoryEntry *)((char *)(entries) + offset); } static ValkeyModuleCommandKeySpec * moduleCmdKeySpecAt(const ValkeyModuleCommandInfoVersion *version, ValkeyModuleCommandKeySpec *keyspecs, int index) { off_t offset = index * version->sizeof_keyspec; return (ValkeyModuleCommandKeySpec *)((char *)(keyspecs) + offset); } static ValkeyModuleCommandArg * moduleCmdArgAt(const ValkeyModuleCommandInfoVersion *version, const ValkeyModuleCommandArg *args, int index) { off_t offset = index * version->sizeof_arg; return (ValkeyModuleCommandArg *)((char *)(args) + offset); } /* Recursively populate the args structure (setting num_args to the number of * subargs) and return the number of args. */ int populateArgsStructure(struct serverCommandArg *args) { if (!args) return 0; int count = 0; while (args->name) { serverAssert(count < INT_MAX); args->num_args = populateArgsStructure(args->subargs); count++; args++; } return count; } /* ValkeyModule_AddACLCategory can be used to add new ACL command categories. Category names * can only contain alphanumeric characters, underscores, or dashes. Categories can only be added * during the ValkeyModule_OnLoad function. Once a category has been added, it can not be removed. * Any module can register a command to any added categories using ValkeyModule_SetCommandACLCategories. * * Returns: * - VALKEYMODULE_OK on successfully adding the new ACL category. * - VALKEYMODULE_ERR on failure. * * On error the errno is set to: * - EINVAL if the name contains invalid characters. * - EBUSY if the category name already exists. * - ENOMEM if the number of categories reached the max limit of 64 categories. */ int VM_AddACLCategory(ValkeyModuleCtx *ctx, const char *name) { if (!ctx->module->onload) { errno = EINVAL; return VALKEYMODULE_ERR; } if (moduleVerifyResourceName(name) == VALKEYMODULE_ERR) { errno = EINVAL; return VALKEYMODULE_ERR; } if (ACLGetCommandCategoryFlagByName(name)) { errno = EBUSY; return VALKEYMODULE_ERR; } if (ACLAddCommandCategory(name, 0)) { ctx->module->num_acl_categories_added++; return VALKEYMODULE_OK; } else { errno = ENOMEM; return VALKEYMODULE_ERR; } } /* Helper for categoryFlagsFromString(). Attempts to find an acl flag representing the provided flag string * and adds that flag to acl_categories_flags if a match is found. * * Returns '1' if acl category flag is recognized or * returns '0' if not recognized */ int matchAclCategoryFlag(char *flag, int64_t *acl_categories_flags) { uint64_t this_flag = ACLGetCommandCategoryFlagByName(flag); if (this_flag) { *acl_categories_flags |= (int64_t)this_flag; return 1; } return 0; /* Unrecognized */ } /* Helper for VM_SetCommandACLCategories(). Turns a string representing acl category * flags into the acl category flags used by the server ACL which allows users to access * the module commands by acl categories. * * It returns the set of acl flags, or -1 if unknown flags are found. */ int64_t categoryFlagsFromString(char *aclflags) { int count, j; int64_t acl_categories_flags = 0; sds *tokens = sdssplitlen(aclflags, strlen(aclflags), " ", 1, &count); for (j = 0; j < count; j++) { char *t = tokens[j]; if (!matchAclCategoryFlag(t, &acl_categories_flags)) { serverLog(LL_WARNING, "Unrecognized categories flag %s on module load", t); break; } } sdsfreesplitres(tokens, count); if (j != count) return -1; /* Some token not processed correctly. */ return acl_categories_flags; } /* ValkeyModule_SetCommandACLCategories can be used to set ACL categories to module * commands and subcommands. The set of ACL categories should be passed as * a space separated C string 'aclflags'. * * Example, the acl flags 'write slow' marks the command as part of the write and * slow ACL categories. * * On success VALKEYMODULE_OK is returned. On error VALKEYMODULE_ERR is returned. * * This function can only be called during the ValkeyModule_OnLoad function. If called * outside of this function, an error is returned. */ int VM_SetCommandACLCategories(ValkeyModuleCommand *command, const char *aclflags) { if (!command || !command->module || !command->module->onload) return VALKEYMODULE_ERR; int64_t categories_flags = aclflags ? categoryFlagsFromString((char *)aclflags) : 0; if (categories_flags == -1) return VALKEYMODULE_ERR; struct serverCommand *rcmd = command->serverCmd; rcmd->acl_categories = categories_flags; /* ACL categories flags for module command */ command->module->num_commands_with_acl_categories++; return VALKEYMODULE_OK; } /* Set additional command information. * * Affects the output of `COMMAND`, `COMMAND INFO` and `COMMAND DOCS`, Cluster, * ACL and is used to filter commands with the wrong number of arguments before * the call reaches the module code. * * This function can be called after creating a command using VM_CreateCommand * and fetching the command pointer using VM_GetCommand. The information can * only be set once for each command and has the following structure: * * typedef struct ValkeyModuleCommandInfo { * const ValkeyModuleCommandInfoVersion *version; * const char *summary; * const char *complexity; * const char *since; * ValkeyModuleCommandHistoryEntry *history; * const char *tips; * int arity; * ValkeyModuleCommandKeySpec *key_specs; * ValkeyModuleCommandArg *args; * } ValkeyModuleCommandInfo; * * All fields except `version` are optional. Explanation of the fields: * * - `version`: This field enables compatibility with different server versions. * Always set this field to VALKEYMODULE_COMMAND_INFO_VERSION. * * - `summary`: A short description of the command (optional). * * - `complexity`: Complexity description (optional). * * - `since`: The version where the command was introduced (optional). * Note: The version specified should be the module's, not the server version. * * - `history`: An array of ValkeyModuleCommandHistoryEntry (optional), which is * a struct with the following fields: * * const char *since; * const char *changes; * * `since` is a version string and `changes` is a string describing the * changes. The array is terminated by a zeroed entry, i.e. an entry with * both strings set to NULL. * * - `tips`: A string of space-separated tips regarding this command, meant for * clients and proxies. See https://valkey.io/topics/command-tips. * * - `arity`: Number of arguments, including the command name itself. A positive * number specifies an exact number of arguments and a negative number * specifies a minimum number of arguments, so use -N to say >= N. The server * validates a call before passing it to a module, so this can replace an * arity check inside the module command implementation. A value of 0 (or an * omitted arity field) is equivalent to -2 if the command has sub commands * and -1 otherwise. * * - `key_specs`: An array of ValkeyModuleCommandKeySpec, terminated by an * element memset to zero. This is a scheme that tries to describe the * positions of key arguments better than the old VM_CreateCommand arguments * `firstkey`, `lastkey`, `keystep` and is needed if those three are not * enough to describe the key positions. There are two steps to retrieve key * positions: *begin search* (BS) in which index should find the first key and * *find keys* (FK) which, relative to the output of BS, describes how can we * will which arguments are keys. Additionally, there are key specific flags. * * Key-specs cause the triplet (firstkey, lastkey, keystep) given in * VM_CreateCommand to be recomputed, but it is still useful to provide * these three parameters in VM_CreateCommand, to better support old server * versions where VM_SetCommandInfo is not available. * * Note that key-specs don't fully replace the "getkeys-api" (see * VM_CreateCommand, VM_IsKeysPositionRequest and VM_KeyAtPosWithFlags) so * it may be a good idea to supply both key-specs and implement the * getkeys-api. * * A key-spec has the following structure: * * typedef struct ValkeyModuleCommandKeySpec { * const char *notes; * uint64_t flags; * ValkeyModuleKeySpecBeginSearchType begin_search_type; * union { * struct { * int pos; * } index; * struct { * const char *keyword; * int startfrom; * } keyword; * } bs; * ValkeyModuleKeySpecFindKeysType find_keys_type; * union { * struct { * int lastkey; * int keystep; * int limit; * } range; * struct { * int keynumidx; * int firstkey; * int keystep; * } keynum; * } fk; * } ValkeyModuleCommandKeySpec; * * Explanation of the fields of ValkeyModuleCommandKeySpec: * * * `notes`: Optional notes or clarifications about this key spec. * * * `flags`: A bitwise or of key-spec flags described below. * * * `begin_search_type`: This describes how the first key is discovered. * There are two ways to determine the first key: * * * `VALKEYMODULE_KSPEC_BS_UNKNOWN`: There is no way to tell where the * key args start. * * `VALKEYMODULE_KSPEC_BS_INDEX`: Key args start at a constant index. * * `VALKEYMODULE_KSPEC_BS_KEYWORD`: Key args start just after a * specific keyword. * * * `bs`: This is a union in which the `index` or `keyword` branch is used * depending on the value of the `begin_search_type` field. * * * `bs.index.pos`: The index from which we start the search for keys. * (`VALKEYMODULE_KSPEC_BS_INDEX` only.) * * * `bs.keyword.keyword`: The keyword (string) that indicates the * beginning of key arguments. (`VALKEYMODULE_KSPEC_BS_KEYWORD` only.) * * * `bs.keyword.startfrom`: An index in argv from which to start * searching. Can be negative, which means start search from the end, * in reverse. Example: -2 means to start in reverse from the * penultimate argument. (`VALKEYMODULE_KSPEC_BS_KEYWORD` only.) * * * `find_keys_type`: After the "begin search", this describes which * arguments are keys. The strategies are: * * * `VALKEYMODULE_KSPEC_BS_UNKNOWN`: There is no way to tell where the * key args are located. * * `VALKEYMODULE_KSPEC_FK_RANGE`: Keys end at a specific index (or * relative to the last argument). * * `VALKEYMODULE_KSPEC_FK_KEYNUM`: There's an argument that contains * the number of key args somewhere before the keys themselves. * * `find_keys_type` and `fk` can be omitted if this keyspec describes * exactly one key. * * * `fk`: This is a union in which the `range` or `keynum` branch is used * depending on the value of the `find_keys_type` field. * * * `fk.range` (for `VALKEYMODULE_KSPEC_FK_RANGE`): A struct with the * following fields: * * * `lastkey`: Index of the last key relative to the result of the * begin search step. Can be negative, in which case it's not * relative. -1 indicates the last argument, -2 one before the * last and so on. * * * `keystep`: How many arguments should we skip after finding a * key, in order to find the next one? * * * `limit`: If `lastkey` is -1, we use `limit` to stop the search * by a factor. 0 and 1 mean no limit. 2 means 1/2 of the * remaining args, 3 means 1/3, and so on. * * * `fk.keynum` (for `VALKEYMODULE_KSPEC_FK_KEYNUM`): A struct with the * following fields: * * * `keynumidx`: Index of the argument containing the number of * keys to come, relative to the result of the begin search step. * * * `firstkey`: Index of the fist key relative to the result of the * begin search step. (Usually it's just after `keynumidx`, in * which case it should be set to `keynumidx + 1`.) * * * `keystep`: How many arguments should we skip after finding a * key, in order to find the next one? * * Key-spec flags: * * The first four refer to what the command actually does with the *value or * metadata of the key*, and not necessarily the user data or how it affects * it. Each key-spec may must have exactly one of these. Any operation * that's not distinctly deletion, overwrite or read-only would be marked as * RW. * * * `VALKEYMODULE_CMD_KEY_RO`: Read-Only. Reads the value of the key, but * doesn't necessarily return it. * * * `VALKEYMODULE_CMD_KEY_RW`: Read-Write. Modifies the data stored in the * value of the key or its metadata. * * * `VALKEYMODULE_CMD_KEY_OW`: Overwrite. Overwrites the data stored in the * value of the key. * * * `VALKEYMODULE_CMD_KEY_RM`: Deletes the key. * * The next four refer to *user data inside the value of the key*, not the * metadata like LRU, type, cardinality. It refers to the logical operation * on the user's data (actual input strings or TTL), being * used/returned/copied/changed. It doesn't refer to modification or * returning of metadata (like type, count, presence of data). ACCESS can be * combined with one of the write operations INSERT, DELETE or UPDATE. Any * write that's not an INSERT or a DELETE would be UPDATE. * * * `VALKEYMODULE_CMD_KEY_ACCESS`: Returns, copies or uses the user data * from the value of the key. * * * `VALKEYMODULE_CMD_KEY_UPDATE`: Updates data to the value, new value may * depend on the old value. * * * `VALKEYMODULE_CMD_KEY_INSERT`: Adds data to the value with no chance of * modification or deletion of existing data. * * * `VALKEYMODULE_CMD_KEY_DELETE`: Explicitly deletes some content from the * value of the key. * * Other flags: * * * `VALKEYMODULE_CMD_KEY_NOT_KEY`: The key is not actually a key, but * should be routed in cluster mode as if it was a key. * * * `VALKEYMODULE_CMD_KEY_INCOMPLETE`: The keyspec might not point out all * the keys it should cover. * * * `VALKEYMODULE_CMD_KEY_VARIABLE_FLAGS`: Some keys might have different * flags depending on arguments. * * - `args`: An array of ValkeyModuleCommandArg, terminated by an element memset * to zero. ValkeyModuleCommandArg is a structure with at the fields described * below. * * typedef struct ValkeyModuleCommandArg { * const char *name; * ValkeyModuleCommandArgType type; * int key_spec_index; * const char *token; * const char *summary; * const char *since; * int flags; * struct ValkeyModuleCommandArg *subargs; * } ValkeyModuleCommandArg; * * Explanation of the fields: * * * `name`: Name of the argument. * * * `type`: The type of the argument. See below for details. The types * `VALKEYMODULE_ARG_TYPE_ONEOF` and `VALKEYMODULE_ARG_TYPE_BLOCK` require * an argument to have sub-arguments, i.e. `subargs`. * * * `key_spec_index`: If the `type` is `VALKEYMODULE_ARG_TYPE_KEY` you must * provide the index of the key-spec associated with this argument. See * `key_specs` above. If the argument is not a key, you may specify -1. * * * `token`: The token preceding the argument (optional). Example: the * argument `seconds` in `SET` has a token `EX`. If the argument consists * of only a token (for example `NX` in `SET`) the type should be * `VALKEYMODULE_ARG_TYPE_PURE_TOKEN` and `value` should be NULL. * * * `summary`: A short description of the argument (optional). * * * `since`: The first version which included this argument (optional). * * * `flags`: A bitwise or of the macros `VALKEYMODULE_CMD_ARG_*`. See below. * * * `value`: The display-value of the argument. This string is what should * be displayed when creating the command syntax from the output of * `COMMAND`. If `token` is not NULL, it should also be displayed. * * Explanation of `ValkeyModuleCommandArgType`: * * * `VALKEYMODULE_ARG_TYPE_STRING`: String argument. * * `VALKEYMODULE_ARG_TYPE_INTEGER`: Integer argument. * * `VALKEYMODULE_ARG_TYPE_DOUBLE`: Double-precision float argument. * * `VALKEYMODULE_ARG_TYPE_KEY`: String argument representing a keyname. * * `VALKEYMODULE_ARG_TYPE_PATTERN`: String, but regex pattern. * * `VALKEYMODULE_ARG_TYPE_UNIX_TIME`: Integer, but Unix timestamp. * * `VALKEYMODULE_ARG_TYPE_PURE_TOKEN`: Argument doesn't have a placeholder. * It's just a token without a value. Example: the `KEEPTTL` option of the * `SET` command. * * `VALKEYMODULE_ARG_TYPE_ONEOF`: Used when the user can choose only one of * a few sub-arguments. Requires `subargs`. Example: the `NX` and `XX` * options of `SET`. * * `VALKEYMODULE_ARG_TYPE_BLOCK`: Used when one wants to group together * several sub-arguments, usually to apply something on all of them, like * making the entire group "optional". Requires `subargs`. Example: the * `LIMIT offset count` parameters in `ZRANGE`. * * Explanation of the command argument flags: * * * `VALKEYMODULE_CMD_ARG_OPTIONAL`: The argument is optional (like GET in * the SET command). * * `VALKEYMODULE_CMD_ARG_MULTIPLE`: The argument may repeat itself (like * key in DEL). * * `VALKEYMODULE_CMD_ARG_MULTIPLE_TOKEN`: The argument may repeat itself, * and so does its token (like `GET pattern` in SORT). * * On success VALKEYMODULE_OK is returned. On error VALKEYMODULE_ERR is returned * and `errno` is set to EINVAL if invalid info was provided or EEXIST if info * has already been set. If the info is invalid, a warning is logged explaining * which part of the info is invalid and why. */ int VM_SetCommandInfo(ValkeyModuleCommand *command, const ValkeyModuleCommandInfo *info) { if (!moduleValidateCommandInfo(info)) { errno = EINVAL; return VALKEYMODULE_ERR; } struct serverCommand *cmd = command->serverCmd; /* Check if any info has already been set. Overwriting info involves freeing * the old info, which is not implemented. */ if (cmd->summary || cmd->complexity || cmd->since || cmd->history || cmd->tips || cmd->args || !(cmd->key_specs_num == 0 || /* Allow key spec populated from legacy (first,last,step) to exist. */ (cmd->key_specs_num == 1 && cmd->key_specs[0].begin_search_type == KSPEC_BS_INDEX && cmd->key_specs[0].find_keys_type == KSPEC_FK_RANGE))) { errno = EEXIST; return VALKEYMODULE_ERR; } if (info->summary) cmd->summary = zstrdup(info->summary); if (info->complexity) cmd->complexity = zstrdup(info->complexity); if (info->since) cmd->since = zstrdup(info->since); const ValkeyModuleCommandInfoVersion *version = info->version; if (info->history) { size_t count = 0; while (moduleCmdHistoryEntryAt(version, info->history, count)->since) count++; serverAssert(count < SIZE_MAX / sizeof(commandHistory)); cmd->history = zmalloc(sizeof(commandHistory) * (count + 1)); for (size_t j = 0; j < count; j++) { ValkeyModuleCommandHistoryEntry *entry = moduleCmdHistoryEntryAt(version, info->history, j); cmd->history[j].since = zstrdup(entry->since); cmd->history[j].changes = zstrdup(entry->changes); } cmd->history[count].since = NULL; cmd->history[count].changes = NULL; cmd->num_history = count; } if (info->tips) { int count; sds *tokens = sdssplitlen(info->tips, strlen(info->tips), " ", 1, &count); if (tokens) { cmd->tips = zmalloc(sizeof(char *) * (count + 1)); for (int j = 0; j < count; j++) { cmd->tips[j] = zstrdup(tokens[j]); } cmd->tips[count] = NULL; cmd->num_tips = count; sdsfreesplitres(tokens, count); } } if (info->arity) cmd->arity = info->arity; if (info->key_specs) { /* Count and allocate the key specs. */ size_t count = 0; while (moduleCmdKeySpecAt(version, info->key_specs, count)->begin_search_type) count++; serverAssert(count < INT_MAX); zfree(cmd->key_specs); cmd->key_specs = zmalloc(sizeof(keySpec) * count); /* Copy the contents of the ValkeyModuleCommandKeySpec array. */ cmd->key_specs_num = count; for (size_t j = 0; j < count; j++) { ValkeyModuleCommandKeySpec *spec = moduleCmdKeySpecAt(version, info->key_specs, j); cmd->key_specs[j].notes = spec->notes ? zstrdup(spec->notes) : NULL; cmd->key_specs[j].flags = moduleConvertKeySpecsFlags(spec->flags, 1); switch (spec->begin_search_type) { case VALKEYMODULE_KSPEC_BS_UNKNOWN: cmd->key_specs[j].begin_search_type = KSPEC_BS_UNKNOWN; break; case VALKEYMODULE_KSPEC_BS_INDEX: cmd->key_specs[j].begin_search_type = KSPEC_BS_INDEX; cmd->key_specs[j].bs.index.pos = spec->bs.index.pos; break; case VALKEYMODULE_KSPEC_BS_KEYWORD: cmd->key_specs[j].begin_search_type = KSPEC_BS_KEYWORD; cmd->key_specs[j].bs.keyword.keyword = zstrdup(spec->bs.keyword.keyword); cmd->key_specs[j].bs.keyword.startfrom = spec->bs.keyword.startfrom; break; default: /* Can't happen; stopped in moduleValidateCommandInfo(). */ serverPanic("Unknown begin_search_type"); } switch (spec->find_keys_type) { case VALKEYMODULE_KSPEC_FK_OMITTED: /* Omitted field is shorthand to say that it's a single key. */ cmd->key_specs[j].find_keys_type = KSPEC_FK_RANGE; cmd->key_specs[j].fk.range.lastkey = 0; cmd->key_specs[j].fk.range.keystep = 1; cmd->key_specs[j].fk.range.limit = 0; break; case VALKEYMODULE_KSPEC_FK_UNKNOWN: cmd->key_specs[j].find_keys_type = KSPEC_FK_UNKNOWN; break; case VALKEYMODULE_KSPEC_FK_RANGE: cmd->key_specs[j].find_keys_type = KSPEC_FK_RANGE; cmd->key_specs[j].fk.range.lastkey = spec->fk.range.lastkey; cmd->key_specs[j].fk.range.keystep = spec->fk.range.keystep; cmd->key_specs[j].fk.range.limit = spec->fk.range.limit; break; case VALKEYMODULE_KSPEC_FK_KEYNUM: cmd->key_specs[j].find_keys_type = KSPEC_FK_KEYNUM; cmd->key_specs[j].fk.keynum.keynumidx = spec->fk.keynum.keynumidx; cmd->key_specs[j].fk.keynum.firstkey = spec->fk.keynum.firstkey; cmd->key_specs[j].fk.keynum.keystep = spec->fk.keynum.keystep; break; default: /* Can't happen; stopped in moduleValidateCommandInfo(). */ serverPanic("Unknown find_keys_type"); } } /* Update the legacy (first,last,step) spec and "movablekeys" flag used by the COMMAND command, * by trying to "glue" consecutive range key specs. */ populateCommandLegacyRangeSpec(cmd); } if (info->args) { cmd->args = moduleCopyCommandArgs(info->args, version); /* Populate arg.num_args with the number of subargs, recursively */ cmd->num_args = populateArgsStructure(cmd->args); } /* Fields added in future versions to be added here, under conditions like * `if (info->version >= 2) { access version 2 fields here }` */ return VALKEYMODULE_OK; } /* Returns 1 if v is a power of two, 0 otherwise. */ static inline int isPowerOfTwo(uint64_t v) { return v && !(v & (v - 1)); } /* Returns 1 if the command info is valid and 0 otherwise. */ static int moduleValidateCommandInfo(const ValkeyModuleCommandInfo *info) { const ValkeyModuleCommandInfoVersion *version = info->version; if (!version) { serverLog(LL_WARNING, "Invalid command info: version missing"); return 0; } /* No validation for the fields summary, complexity, since, tips (strings or * NULL) and arity (any integer). */ /* History: If since is set, changes must also be set. */ if (info->history) { for (size_t j = 0; moduleCmdHistoryEntryAt(version, info->history, j)->since; j++) { if (!moduleCmdHistoryEntryAt(version, info->history, j)->changes) { serverLog(LL_WARNING, "Invalid command info: history[%zd].changes missing", j); return 0; } } } /* Key specs. */ if (info->key_specs) { for (size_t j = 0; moduleCmdKeySpecAt(version, info->key_specs, j)->begin_search_type; j++) { ValkeyModuleCommandKeySpec *spec = moduleCmdKeySpecAt(version, info->key_specs, j); if (j >= INT_MAX) { serverLog(LL_WARNING, "Invalid command info: Too many key specs"); return 0; /* serverCommand.key_specs_num is an int. */ } /* Flags. Exactly one flag in a group is set if and only if the * masked bits is a power of two. */ uint64_t key_flags = VALKEYMODULE_CMD_KEY_RO | VALKEYMODULE_CMD_KEY_RW | VALKEYMODULE_CMD_KEY_OW | VALKEYMODULE_CMD_KEY_RM; uint64_t write_flags = VALKEYMODULE_CMD_KEY_INSERT | VALKEYMODULE_CMD_KEY_DELETE | VALKEYMODULE_CMD_KEY_UPDATE; if (!isPowerOfTwo(spec->flags & key_flags)) { serverLog(LL_WARNING, "Invalid command info: key_specs[%zd].flags: " "Exactly one of the flags RO, RW, OW, RM required", j); return 0; } if ((spec->flags & write_flags) != 0 && !isPowerOfTwo(spec->flags & write_flags)) { serverLog(LL_WARNING, "Invalid command info: key_specs[%zd].flags: " "INSERT, DELETE and UPDATE are mutually exclusive", j); return 0; } switch (spec->begin_search_type) { case VALKEYMODULE_KSPEC_BS_UNKNOWN: break; case VALKEYMODULE_KSPEC_BS_INDEX: break; case VALKEYMODULE_KSPEC_BS_KEYWORD: if (spec->bs.keyword.keyword == NULL) { serverLog(LL_WARNING, "Invalid command info: key_specs[%zd].bs.keyword.keyword " "required when begin_search_type is KEYWORD", j); return 0; } break; default: serverLog(LL_WARNING, "Invalid command info: key_specs[%zd].begin_search_type: " "Invalid value %d", j, spec->begin_search_type); return 0; } /* Validate find_keys_type. */ switch (spec->find_keys_type) { case VALKEYMODULE_KSPEC_FK_OMITTED: break; /* short for RANGE {0,1,0} */ case VALKEYMODULE_KSPEC_FK_UNKNOWN: break; case VALKEYMODULE_KSPEC_FK_RANGE: break; case VALKEYMODULE_KSPEC_FK_KEYNUM: break; default: serverLog(LL_WARNING, "Invalid command info: key_specs[%zd].find_keys_type: " "Invalid value %d", j, spec->find_keys_type); return 0; } } } /* Args, subargs (recursive) */ return moduleValidateCommandArgs(info->args, version); } /* When from_api is true, converts from VALKEYMODULE_CMD_KEY_* flags to CMD_KEY_* flags. * When from_api is false, converts from CMD_KEY_* flags to VALKEYMODULE_CMD_KEY_* flags. */ static int64_t moduleConvertKeySpecsFlags(int64_t flags, int from_api) { int64_t out = 0; int64_t map[][2] = {{VALKEYMODULE_CMD_KEY_RO, CMD_KEY_RO}, {VALKEYMODULE_CMD_KEY_RW, CMD_KEY_RW}, {VALKEYMODULE_CMD_KEY_OW, CMD_KEY_OW}, {VALKEYMODULE_CMD_KEY_RM, CMD_KEY_RM}, {VALKEYMODULE_CMD_KEY_ACCESS, CMD_KEY_ACCESS}, {VALKEYMODULE_CMD_KEY_INSERT, CMD_KEY_INSERT}, {VALKEYMODULE_CMD_KEY_UPDATE, CMD_KEY_UPDATE}, {VALKEYMODULE_CMD_KEY_DELETE, CMD_KEY_DELETE}, {VALKEYMODULE_CMD_KEY_NOT_KEY, CMD_KEY_NOT_KEY}, {VALKEYMODULE_CMD_KEY_INCOMPLETE, CMD_KEY_INCOMPLETE}, {VALKEYMODULE_CMD_KEY_VARIABLE_FLAGS, CMD_KEY_VARIABLE_FLAGS}, {0, 0}}; int from_idx = from_api ? 0 : 1, to_idx = !from_idx; for (int i = 0; map[i][0]; i++) if (flags & map[i][from_idx]) out |= map[i][to_idx]; return out; } /* Validates an array of ValkeyModuleCommandArg. Returns 1 if it's valid and 0 if * it's invalid. */ static int moduleValidateCommandArgs(ValkeyModuleCommandArg *args, const ValkeyModuleCommandInfoVersion *version) { if (args == NULL) return 1; /* Missing args is OK. */ for (size_t j = 0; moduleCmdArgAt(version, args, j)->name != NULL; j++) { ValkeyModuleCommandArg *arg = moduleCmdArgAt(version, args, j); int arg_type_error = 0; moduleConvertArgType(arg->type, &arg_type_error); if (arg_type_error) { serverLog(LL_WARNING, "Invalid command info: Argument \"%s\": Undefined type %d", arg->name, arg->type); return 0; } if (arg->type == VALKEYMODULE_ARG_TYPE_PURE_TOKEN && !arg->token) { serverLog(LL_WARNING, "Invalid command info: Argument \"%s\": " "token required when type is PURE_TOKEN", args[j].name); return 0; } if (arg->type == VALKEYMODULE_ARG_TYPE_KEY) { if (arg->key_spec_index < 0) { serverLog(LL_WARNING, "Invalid command info: Argument \"%s\": " "key_spec_index required when type is KEY", arg->name); return 0; } } else if (arg->key_spec_index != -1 && arg->key_spec_index != 0) { /* 0 is allowed for convenience, to allow it to be omitted in * compound struct literals on the form `.field = value`. */ serverLog(LL_WARNING, "Invalid command info: Argument \"%s\": " "key_spec_index specified but type isn't KEY", arg->name); return 0; } if (arg->flags & ~(_VALKEYMODULE_CMD_ARG_NEXT - 1)) { serverLog(LL_WARNING, "Invalid command info: Argument \"%s\": Invalid flags", arg->name); return 0; } if (arg->type == VALKEYMODULE_ARG_TYPE_ONEOF || arg->type == VALKEYMODULE_ARG_TYPE_BLOCK) { if (arg->subargs == NULL) { serverLog(LL_WARNING, "Invalid command info: Argument \"%s\": " "subargs required when type is ONEOF or BLOCK", arg->name); return 0; } if (!moduleValidateCommandArgs(arg->subargs, version)) return 0; } else { if (arg->subargs != NULL) { serverLog(LL_WARNING, "Invalid command info: Argument \"%s\": " "subargs specified but type isn't ONEOF nor BLOCK", arg->name); return 0; } } } return 1; } /* Converts an array of ValkeyModuleCommandArg into a freshly allocated array of * struct serverCommandArg. */ static struct serverCommandArg *moduleCopyCommandArgs(ValkeyModuleCommandArg *args, const ValkeyModuleCommandInfoVersion *version) { size_t count = 0; while (moduleCmdArgAt(version, args, count)->name) count++; serverAssert(count < SIZE_MAX / sizeof(struct serverCommandArg)); struct serverCommandArg *realargs = zcalloc((count + 1) * sizeof(serverCommandArg)); for (size_t j = 0; j < count; j++) { ValkeyModuleCommandArg *arg = moduleCmdArgAt(version, args, j); realargs[j].name = zstrdup(arg->name); realargs[j].type = moduleConvertArgType(arg->type, NULL); if (arg->type == VALKEYMODULE_ARG_TYPE_KEY) realargs[j].key_spec_index = arg->key_spec_index; else realargs[j].key_spec_index = -1; if (arg->token) realargs[j].token = zstrdup(arg->token); if (arg->summary) realargs[j].summary = zstrdup(arg->summary); if (arg->since) realargs[j].since = zstrdup(arg->since); if (arg->deprecated_since) realargs[j].deprecated_since = zstrdup(arg->deprecated_since); if (arg->display_text) realargs[j].display_text = zstrdup(arg->display_text); realargs[j].flags = moduleConvertArgFlags(arg->flags); if (arg->subargs) realargs[j].subargs = moduleCopyCommandArgs(arg->subargs, version); } return realargs; } static serverCommandArgType moduleConvertArgType(ValkeyModuleCommandArgType type, int *error) { if (error) *error = 0; switch (type) { case VALKEYMODULE_ARG_TYPE_STRING: return ARG_TYPE_STRING; case VALKEYMODULE_ARG_TYPE_INTEGER: return ARG_TYPE_INTEGER; case VALKEYMODULE_ARG_TYPE_DOUBLE: return ARG_TYPE_DOUBLE; case VALKEYMODULE_ARG_TYPE_KEY: return ARG_TYPE_KEY; case VALKEYMODULE_ARG_TYPE_PATTERN: return ARG_TYPE_PATTERN; case VALKEYMODULE_ARG_TYPE_UNIX_TIME: return ARG_TYPE_UNIX_TIME; case VALKEYMODULE_ARG_TYPE_PURE_TOKEN: return ARG_TYPE_PURE_TOKEN; case VALKEYMODULE_ARG_TYPE_ONEOF: return ARG_TYPE_ONEOF; case VALKEYMODULE_ARG_TYPE_BLOCK: return ARG_TYPE_BLOCK; default: if (error) *error = 1; return -1; } } static int moduleConvertArgFlags(int flags) { int realflags = 0; if (flags & VALKEYMODULE_CMD_ARG_OPTIONAL) realflags |= CMD_ARG_OPTIONAL; if (flags & VALKEYMODULE_CMD_ARG_MULTIPLE) realflags |= CMD_ARG_MULTIPLE; if (flags & VALKEYMODULE_CMD_ARG_MULTIPLE_TOKEN) realflags |= CMD_ARG_MULTIPLE_TOKEN; return realflags; } /* Return `struct ValkeyModule *` as `void *` to avoid exposing it outside of module.c. */ void *moduleGetHandleByName(char *modulename) { return dictFetchValue(modules, modulename); } /* Returns 1 if `cmd` is a command of the module `modulename`. 0 otherwise. */ int moduleIsModuleCommand(void *module_handle, struct serverCommand *cmd) { if (cmd->proc != ValkeyModuleCommandDispatcher) return 0; if (module_handle == NULL) return 0; ValkeyModuleCommand *cp = cmd->module_cmd; return (cp->module == module_handle); } /* ValkeyModule_UpdateRuntimeArgs can be used to update the module argument values. * The function parameter 'argc' indicates the number of updated arguments, and 'argv' * represents the values of the updated arguments. * Once 'CONFIG REWRITE' command is called, the updated argument values can be saved into conf file. * * The function always returns VALKEYMODULE_OK. */ int VM_UpdateRuntimeArgs(ValkeyModuleCtx *ctx, ValkeyModuleString **argv, int argc) { struct moduleLoadQueueEntry *loadmod = ctx->module->loadmod; for (int i = 0; i < loadmod->argc; i++) { decrRefCount(loadmod->argv[i]); } zfree(loadmod->argv); loadmod->argv = argc - 1 ? zmalloc(sizeof(robj *) * (argc - 1)) : NULL; loadmod->argc = argc - 1; for (int i = 1; i < argc; i++) { loadmod->argv[i - 1] = argv[i]; incrRefCount(loadmod->argv[i - 1]); } return VALKEYMODULE_OK; } /* -------------------------------------------------------------------------- * ## Module information and time measurement * -------------------------------------------------------------------------- */ int moduleListConfigMatch(void *config, void *name) { return strcasecmp(((ModuleConfig *)config)->name, (char *)name) == 0; } void moduleListFree(void *config) { ModuleConfig *module_config = (ModuleConfig *)config; sdsfree(module_config->name); zfree(config); } void VM_SetModuleAttribs(ValkeyModuleCtx *ctx, const char *name, int ver, int apiver) { /* Called by VM_Init() to setup the `ctx->module` structure. * * This is an internal function, module developers don't need * to use it. */ ValkeyModule *module; if (ctx->module != NULL) return; module = zmalloc(sizeof(*module)); module->name = sdsnew(name); module->ver = ver; module->apiver = apiver; module->types = listCreate(); module->usedby = listCreate(); module->using = listCreate(); module->filters = listCreate(); module->module_configs = listCreate(); listSetMatchMethod(module->module_configs, moduleListConfigMatch); listSetFreeMethod(module->module_configs, moduleListFree); module->in_call = 0; module->configs_initialized = 0; module->in_hook = 0; module->options = 0; module->info_cb = 0; module->defrag_cb = 0; module->loadmod = NULL; module->num_commands_with_acl_categories = 0; module->onload = 1; module->num_acl_categories_added = 0; ctx->module = module; } /* Return non-zero if the module name is busy. * Otherwise zero is returned. */ int VM_IsModuleNameBusy(const char *name) { sds modulename = sdsnew(name); dictEntry *de = dictFind(modules, modulename); sdsfree(modulename); return de != NULL; } /* Return the current UNIX time in milliseconds. */ mstime_t VM_Milliseconds(void) { return mstime(); } /* Return counter of micro-seconds relative to an arbitrary point in time. */ uint64_t VM_MonotonicMicroseconds(void) { return getMonotonicUs(); } /* Return the current UNIX time in microseconds */ ustime_t VM_Microseconds(void) { return ustime(); } /* Return the cached UNIX time in microseconds. * It is updated in the server cron job and before executing a command. * It is useful for complex call stacks, such as a command causing a * key space notification, causing a module to execute a ValkeyModule_Call, * causing another notification, etc. * It makes sense that all this callbacks would use the same clock. */ ustime_t VM_CachedMicroseconds(void) { return server.ustime; } /* Mark a point in time that will be used as the start time to calculate * the elapsed execution time when VM_BlockedClientMeasureTimeEnd() is called. * Within the same command, you can call multiple times * VM_BlockedClientMeasureTimeStart() and VM_BlockedClientMeasureTimeEnd() * to accumulate independent time intervals to the background duration. * This method always return VALKEYMODULE_OK. * * This function is not thread safe, If used in module thread and blocked callback (possibly main thread) * simultaneously, it's recommended to protect them with lock owned by caller instead of GIL. */ int VM_BlockedClientMeasureTimeStart(ValkeyModuleBlockedClient *bc) { elapsedStart(&(bc->background_timer)); return VALKEYMODULE_OK; } /* Mark a point in time that will be used as the end time * to calculate the elapsed execution time. * On success VALKEYMODULE_OK is returned. * This method only returns VALKEYMODULE_ERR if no start time was * previously defined ( meaning VM_BlockedClientMeasureTimeStart was not called ). * * This function is not thread safe, If used in module thread and blocked callback (possibly main thread) * simultaneously, it's recommended to protect them with lock owned by caller instead of GIL. */ int VM_BlockedClientMeasureTimeEnd(ValkeyModuleBlockedClient *bc) { // If the counter is 0 then we haven't called VM_BlockedClientMeasureTimeStart if (!bc->background_timer) return VALKEYMODULE_ERR; bc->background_duration += elapsedUs(bc->background_timer); return VALKEYMODULE_OK; } /* This API allows modules to let the server process background tasks, and some * commands during long blocking execution of a module command. * The module can call this API periodically. * The flags is a bit mask of these: * * - `VALKEYMODULE_YIELD_FLAG_NONE`: No special flags, can perform some background * operations, but not process client commands. * - `VALKEYMODULE_YIELD_FLAG_CLIENTS`: The server can also process client commands. * * The `busy_reply` argument is optional, and can be used to control the verbose * error string after the `-BUSY` error code. * * When the `VALKEYMODULE_YIELD_FLAG_CLIENTS` is used, the server will only start * processing client commands after the time defined by the * `busy-reply-threshold` config, in which case the server will start rejecting most * commands with `-BUSY` error, but allow the ones marked with the `allow-busy` * flag to be executed. * This API can also be used in thread safe context (while locked), and during * loading (in the `rdb_load` callback, in which case it'll reject commands with * the -LOADING error) */ void VM_Yield(ValkeyModuleCtx *ctx, int flags, const char *busy_reply) { static int yield_nesting = 0; /* Avoid nested calls to VM_Yield */ if (yield_nesting) return; yield_nesting++; long long now = getMonotonicUs(); if (now >= ctx->next_yield_time) { /* In loading mode, there's no need to handle busy_module_yield_reply, * and busy_module_yield_flags, since the server is anyway rejecting all * commands with -LOADING. */ if (server.loading) { /* Let the server process events */ processEventsWhileBlocked(); } else { const char *prev_busy_module_yield_reply = server.busy_module_yield_reply; server.busy_module_yield_reply = busy_reply; /* start the blocking operation if not already started. */ if (!server.busy_module_yield_flags) { server.busy_module_yield_flags = BUSY_MODULE_YIELD_EVENTS; blockingOperationStarts(); if (server.current_client) protectClient(server.current_client); } if (flags & VALKEYMODULE_YIELD_FLAG_CLIENTS) server.busy_module_yield_flags |= BUSY_MODULE_YIELD_CLIENTS; /* Let the server process events */ if (!pthread_equal(server.main_thread_id, pthread_self())) { /* If we are not in the main thread, we defer event loop processing to the main thread * after the main thread enters acquiring GIL state in order to protect the event * loop (ae.c) and avoid potential race conditions. */ int acquiring = atomic_load_explicit(&server.module_gil_acquiring, memory_order_relaxed); if (!acquiring) { /* If the main thread has not yet entered the acquiring GIL state, * we attempt to wake it up and exit without waiting for it to * acquire the GIL. This avoids blocking the caller, allowing them to * continue with unfinished tasks before the next yield. * We assume the caller keeps the GIL locked. */ if (write(server.module_pipe[1], "A", 1) != 1) { /* Ignore the error, this is best-effort. */ } } else { /* Release the GIL, yielding CPU to give the main thread an opportunity to start * event processing, and then acquire the GIL again until the main thread releases it. */ moduleReleaseGIL(); sched_yield(); moduleAcquireGIL(); } } else { /* If we are in the main thread, we can safely process events. */ processEventsWhileBlocked(); } server.busy_module_yield_reply = prev_busy_module_yield_reply; /* Possibly restore the previous flags in case of two nested contexts * that use this API with different flags, but keep the first bit * (PROCESS_EVENTS) set, so we know to call blockingOperationEnds on time. */ server.busy_module_yield_flags &= ~BUSY_MODULE_YIELD_CLIENTS; } /* decide when the next event should fire. */ ctx->next_yield_time = now + 1000000 / server.hz; } yield_nesting--; } /* Set flags defining capabilities or behavior bit flags. * * VALKEYMODULE_OPTIONS_HANDLE_IO_ERRORS: * Generally, modules don't need to bother with this, as the process will just * terminate if a read error happens, however, setting this flag would allow * repl-diskless-load to work if enabled. * The module should use ValkeyModule_IsIOError after reads, before using the * data that was read, and in case of error, propagate it upwards, and also be * able to release the partially populated value and all it's allocations. * * VALKEYMODULE_OPTION_NO_IMPLICIT_SIGNAL_MODIFIED: * See VM_SignalModifiedKey(). * * VALKEYMODULE_OPTIONS_HANDLE_REPL_ASYNC_LOAD: * Setting this flag indicates module awareness of diskless async replication (repl-diskless-load=swapdb) * and that the server could be serving reads during replication instead of blocking with LOADING status. * * VALKEYMODULE_OPTIONS_ALLOW_NESTED_KEYSPACE_NOTIFICATIONS: * Declare that the module wants to get nested key-space notifications. * By default, the server will not fire key-space notifications that happened inside * a key-space notification callback. This flag allows to change this behavior * and fire nested key-space notifications. Notice: if enabled, the module * should protected itself from infinite recursion. */ void VM_SetModuleOptions(ValkeyModuleCtx *ctx, int options) { ctx->module->options = options; } /* Signals that the key is modified from user's perspective (i.e. invalidate WATCH * and client side caching). * * This is done automatically when a key opened for writing is closed, unless * the option VALKEYMODULE_OPTION_NO_IMPLICIT_SIGNAL_MODIFIED has been set using * VM_SetModuleOptions(). */ int VM_SignalModifiedKey(ValkeyModuleCtx *ctx, ValkeyModuleString *keyname) { signalModifiedKey(ctx->client, ctx->client->db, keyname); return VALKEYMODULE_OK; } /* -------------------------------------------------------------------------- * ## Automatic memory management for modules * -------------------------------------------------------------------------- */ /* Enable automatic memory management. * * The function must be called as the first function of a command implementation * that wants to use automatic memory. * * When enabled, automatic memory management tracks and automatically frees * keys, call replies and ValkeyModuleString objects once the command returns. In most * cases this eliminates the need of calling the following functions: * * 1. ValkeyModule_CloseKey() * 2. ValkeyModule_FreeCallReply() * 3. ValkeyModule_FreeString() * * These functions can still be used with automatic memory management enabled, * to optimize loops that make numerous allocations for example. */ void VM_AutoMemory(ValkeyModuleCtx *ctx) { ctx->flags |= VALKEYMODULE_CTX_AUTO_MEMORY; } /* Add a new object to release automatically when the callback returns. */ void autoMemoryAdd(ValkeyModuleCtx *ctx, int type, void *ptr) { if (!(ctx->flags & VALKEYMODULE_CTX_AUTO_MEMORY)) return; if (ctx->amqueue_used == ctx->amqueue_len) { ctx->amqueue_len *= 2; if (ctx->amqueue_len < 16) ctx->amqueue_len = 16; ctx->amqueue = zrealloc(ctx->amqueue, sizeof(struct AutoMemEntry) * ctx->amqueue_len); } ctx->amqueue[ctx->amqueue_used].type = type; ctx->amqueue[ctx->amqueue_used].ptr = ptr; ctx->amqueue_used++; } /* Mark an object as freed in the auto release queue, so that users can still * free things manually if they want. * * The function returns 1 if the object was actually found in the auto memory * pool, otherwise 0 is returned. */ int autoMemoryFreed(ValkeyModuleCtx *ctx, int type, void *ptr) { if (!(ctx->flags & VALKEYMODULE_CTX_AUTO_MEMORY)) return 0; int count = (ctx->amqueue_used + 1) / 2; for (int j = 0; j < count; j++) { for (int side = 0; side < 2; side++) { /* For side = 0 check right side of the array, for * side = 1 check the left side instead (zig-zag scanning). */ int i = (side == 0) ? (ctx->amqueue_used - 1 - j) : j; if (ctx->amqueue[i].type == type && ctx->amqueue[i].ptr == ptr) { ctx->amqueue[i].type = VALKEYMODULE_AM_FREED; /* Switch the freed element and the last element, to avoid growing * the queue unnecessarily if we allocate/free in a loop */ if (i != ctx->amqueue_used - 1) { ctx->amqueue[i] = ctx->amqueue[ctx->amqueue_used - 1]; } /* Reduce the size of the queue because we either moved the top * element elsewhere or freed it */ ctx->amqueue_used--; return 1; } } } return 0; } /* Release all the objects in queue. */ void autoMemoryCollect(ValkeyModuleCtx *ctx) { if (!(ctx->flags & VALKEYMODULE_CTX_AUTO_MEMORY)) return; /* Clear the AUTO_MEMORY flag from the context, otherwise the functions * we call to free the resources, will try to scan the auto release * queue to mark the entries as freed. */ ctx->flags &= ~VALKEYMODULE_CTX_AUTO_MEMORY; int j; for (j = 0; j < ctx->amqueue_used; j++) { void *ptr = ctx->amqueue[j].ptr; switch (ctx->amqueue[j].type) { case VALKEYMODULE_AM_STRING: decrRefCount(ptr); break; case VALKEYMODULE_AM_REPLY: VM_FreeCallReply(ptr); break; case VALKEYMODULE_AM_KEY: VM_CloseKey(ptr); break; case VALKEYMODULE_AM_DICT: VM_FreeDict(NULL, ptr); break; case VALKEYMODULE_AM_INFO: VM_FreeServerInfo(NULL, ptr); break; } } ctx->flags |= VALKEYMODULE_CTX_AUTO_MEMORY; zfree(ctx->amqueue); ctx->amqueue = NULL; ctx->amqueue_len = 0; ctx->amqueue_used = 0; } /* -------------------------------------------------------------------------- * ## String objects APIs * -------------------------------------------------------------------------- */ /* Create a new module string object. The returned string must be freed * with ValkeyModule_FreeString(), unless automatic memory is enabled. * * The string is created by copying the `len` bytes starting * at `ptr`. No reference is retained to the passed buffer. * * The module context 'ctx' is optional and may be NULL if you want to create * a string out of the context scope. However in that case, the automatic * memory management will not be available, and the string memory must be * managed manually. */ ValkeyModuleString *VM_CreateString(ValkeyModuleCtx *ctx, const char *ptr, size_t len) { ValkeyModuleString *o = createStringObject(ptr, len); if (ctx != NULL) autoMemoryAdd(ctx, VALKEYMODULE_AM_STRING, o); return o; } /* Create a new module string object from a printf format and arguments. * The returned string must be freed with ValkeyModule_FreeString(), unless * automatic memory is enabled. * * The string is created using the sds formatter function sdscatvprintf(). * * The passed context 'ctx' may be NULL if necessary, see the * ValkeyModule_CreateString() documentation for more info. */ ValkeyModuleString *VM_CreateStringPrintf(ValkeyModuleCtx *ctx, const char *fmt, ...) { sds s = sdsempty(); va_list ap; va_start(ap, fmt); s = sdscatvprintf(s, fmt, ap); va_end(ap); ValkeyModuleString *o = createObject(OBJ_STRING, s); if (ctx != NULL) autoMemoryAdd(ctx, VALKEYMODULE_AM_STRING, o); return o; } /* Like ValkeyModule_CreateString(), but creates a string starting from a `long long` * integer instead of taking a buffer and its length. * * The returned string must be released with ValkeyModule_FreeString() or by * enabling automatic memory management. * * The passed context 'ctx' may be NULL if necessary, see the * ValkeyModule_CreateString() documentation for more info. */ ValkeyModuleString *VM_CreateStringFromLongLong(ValkeyModuleCtx *ctx, long long ll) { char buf[LONG_STR_SIZE]; size_t len = ll2string(buf, sizeof(buf), ll); return VM_CreateString(ctx, buf, len); } /* Like ValkeyModule_CreateString(), but creates a string starting from a `unsigned long long` * integer instead of taking a buffer and its length. * * The returned string must be released with ValkeyModule_FreeString() or by * enabling automatic memory management. * * The passed context 'ctx' may be NULL if necessary, see the * ValkeyModule_CreateString() documentation for more info. */ ValkeyModuleString *VM_CreateStringFromULongLong(ValkeyModuleCtx *ctx, unsigned long long ull) { char buf[LONG_STR_SIZE]; size_t len = ull2string(buf, sizeof(buf), ull); return VM_CreateString(ctx, buf, len); } /* Like ValkeyModule_CreateString(), but creates a string starting from a double * instead of taking a buffer and its length. * * The returned string must be released with ValkeyModule_FreeString() or by * enabling automatic memory management. */ ValkeyModuleString *VM_CreateStringFromDouble(ValkeyModuleCtx *ctx, double d) { char buf[MAX_D2STRING_CHARS]; size_t len = d2string(buf, sizeof(buf), d); return VM_CreateString(ctx, buf, len); } /* Like ValkeyModule_CreateString(), but creates a string starting from a long * double. * * The returned string must be released with ValkeyModule_FreeString() or by * enabling automatic memory management. * * The passed context 'ctx' may be NULL if necessary, see the * ValkeyModule_CreateString() documentation for more info. */ ValkeyModuleString *VM_CreateStringFromLongDouble(ValkeyModuleCtx *ctx, long double ld, int humanfriendly) { char buf[MAX_LONG_DOUBLE_CHARS]; size_t len = ld2string(buf, sizeof(buf), ld, (humanfriendly ? LD_STR_HUMAN : LD_STR_AUTO)); return VM_CreateString(ctx, buf, len); } /* Like ValkeyModule_CreateString(), but creates a string starting from another * ValkeyModuleString. * * The returned string must be released with ValkeyModule_FreeString() or by * enabling automatic memory management. * * The passed context 'ctx' may be NULL if necessary, see the * ValkeyModule_CreateString() documentation for more info. */ ValkeyModuleString *VM_CreateStringFromString(ValkeyModuleCtx *ctx, const ValkeyModuleString *str) { ValkeyModuleString *o = dupStringObject(str); if (ctx != NULL) autoMemoryAdd(ctx, VALKEYMODULE_AM_STRING, o); return o; } /* Creates a string from a stream ID. The returned string must be released with * ValkeyModule_FreeString(), unless automatic memory is enabled. * * The passed context `ctx` may be NULL if necessary. See the * ValkeyModule_CreateString() documentation for more info. */ ValkeyModuleString *VM_CreateStringFromStreamID(ValkeyModuleCtx *ctx, const ValkeyModuleStreamID *id) { streamID streamid = {id->ms, id->seq}; ValkeyModuleString *o = createObjectFromStreamID(&streamid); if (ctx != NULL) autoMemoryAdd(ctx, VALKEYMODULE_AM_STRING, o); return o; } /* Free a module string object obtained with one of the module API calls * that return new string objects. * * It is possible to call this function even when automatic memory management * is enabled. In that case the string will be released ASAP and removed * from the pool of string to release at the end. * * If the string was created with a NULL context 'ctx', it is also possible to * pass ctx as NULL when releasing the string (but passing a context will not * create any issue). Strings created with a context should be freed also passing * the context, so if you want to free a string out of context later, make sure * to create it using a NULL context. * * This API is not thread safe, access to these retained strings (if they originated * from a client command arguments) must be done with GIL locked. */ void VM_FreeString(ValkeyModuleCtx *ctx, ValkeyModuleString *str) { decrRefCount(str); if (ctx != NULL) autoMemoryFreed(ctx, VALKEYMODULE_AM_STRING, str); } /* Every call to this function, will make the string 'str' requiring * an additional call to ValkeyModule_FreeString() in order to really * free the string. Note that the automatic freeing of the string obtained * enabling modules automatic memory management counts for one * ValkeyModule_FreeString() call (it is just executed automatically). * * Normally you want to call this function when, at the same time * the following conditions are true: * * 1. You have automatic memory management enabled. * 2. You want to create string objects. * 3. Those string objects you create need to live *after* the callback * function(for example a command implementation) creating them returns. * * Usually you want this in order to store the created string object * into your own data structure, for example when implementing a new data * type. * * Note that when memory management is turned off, you don't need * any call to RetainString() since creating a string will always result * into a string that lives after the callback function returns, if * no FreeString() call is performed. * * It is possible to call this function with a NULL context. * * When strings are going to be retained for an extended duration, it is good * practice to also call ValkeyModule_TrimStringAllocation() in order to * optimize memory usage. * * Threaded modules that reference retained strings from other threads *must* * explicitly trim the allocation as soon as the string is retained. Not doing * so may result with automatic trimming which is not thread safe. * * This API is not thread safe, access to these retained strings (if they originated * from a client command arguments) must be done with GIL locked. */ void VM_RetainString(ValkeyModuleCtx *ctx, ValkeyModuleString *str) { if (ctx == NULL || !autoMemoryFreed(ctx, VALKEYMODULE_AM_STRING, str)) { /* Increment the string reference counting only if we can't * just remove the object from the list of objects that should * be reclaimed. Why we do that, instead of just incrementing * the refcount in any case, and let the automatic FreeString() * call at the end to bring the refcount back at the desired * value? Because this way we ensure that the object refcount * value is 1 (instead of going to 2 to be dropped later to 1) * after the call to this function. This is needed for functions * like ValkeyModule_StringAppendBuffer() to work. */ incrRefCount(str); } } /** * This function can be used instead of ValkeyModule_RetainString(). * The main difference between the two is that this function will always * succeed, whereas ValkeyModule_RetainString() may fail because of an * assertion. * * The function returns a pointer to ValkeyModuleString, which is owned * by the caller. It requires a call to ValkeyModule_FreeString() to free * the string when automatic memory management is disabled for the context. * When automatic memory management is enabled, you can either call * ValkeyModule_FreeString() or let the automation free it. * * This function is more efficient than ValkeyModule_CreateStringFromString() * because whenever possible, it avoids copying the underlying * ValkeyModuleString. The disadvantage of using this function is that it * might not be possible to use ValkeyModule_StringAppendBuffer() on the * returned ValkeyModuleString. * * It is possible to call this function with a NULL context. * * When strings are going to be held for an extended duration, it is good * practice to also call ValkeyModule_TrimStringAllocation() in order to * optimize memory usage. * * Threaded modules that reference held strings from other threads *must* * explicitly trim the allocation as soon as the string is held. Not doing * so may result with automatic trimming which is not thread safe. * * This API is not thread safe, access to these retained strings (if they originated * from a client command arguments) must be done with GIL locked. */ ValkeyModuleString *VM_HoldString(ValkeyModuleCtx *ctx, ValkeyModuleString *str) { if (str->refcount == OBJ_STATIC_REFCOUNT) { return VM_CreateStringFromString(ctx, str); } incrRefCount(str); if (ctx != NULL) { /* * Put the str in the auto memory management of the ctx. * It might already be there, in this case, the ref count will * be 2 and we will decrease the ref count twice and free the * object in the auto memory free function. * * Why we can not do the same trick of just remove the object * from the auto memory (like in VM_RetainString)? * This code shows the issue: * * VM_AutoMemory(ctx); * str1 = VM_CreateString(ctx, "test", 4); * str2 = VM_HoldString(ctx, str1); * VM_FreeString(str1); * VM_FreeString(str2); * * If after the VM_HoldString we would just remove the string from * the auto memory, this example will cause access to a freed memory * on 'VM_FreeString(str2);' because the String will be free * on 'VM_FreeString(str1);'. * * So it's safer to just increase the ref count * and add the String to auto memory again. * * The limitation is that it is not possible to use ValkeyModule_StringAppendBuffer * on the String. */ autoMemoryAdd(ctx, VALKEYMODULE_AM_STRING, str); } return str; } /* Given a string module object, this function returns the string pointer * and length of the string. The returned pointer and length should only * be used for read only accesses and never modified. */ const char *VM_StringPtrLen(const ValkeyModuleString *str, size_t *len) { if (str == NULL) { const char *errmsg = "(NULL string reply referenced in module)"; if (len) *len = strlen(errmsg); return errmsg; } if (len) *len = sdslen(str->ptr); return str->ptr; } /* -------------------------------------------------------------------------- * Higher level string operations * ------------------------------------------------------------------------- */ /* Convert the string into a `long long` integer, storing it at `*ll`. * Returns VALKEYMODULE_OK on success. If the string can't be parsed * as a valid, strict `long long` (no spaces before/after), VALKEYMODULE_ERR * is returned. */ int VM_StringToLongLong(const ValkeyModuleString *str, long long *ll) { return string2ll(str->ptr, sdslen(str->ptr), ll) ? VALKEYMODULE_OK : VALKEYMODULE_ERR; } /* Convert the string into a `unsigned long long` integer, storing it at `*ull`. * Returns VALKEYMODULE_OK on success. If the string can't be parsed * as a valid, strict `unsigned long long` (no spaces before/after), VALKEYMODULE_ERR * is returned. */ int VM_StringToULongLong(const ValkeyModuleString *str, unsigned long long *ull) { return string2ull(str->ptr, ull) ? VALKEYMODULE_OK : VALKEYMODULE_ERR; } /* Convert the string into a double, storing it at `*d`. * Returns VALKEYMODULE_OK on success or VALKEYMODULE_ERR if the string is * not a valid string representation of a double value. */ int VM_StringToDouble(const ValkeyModuleString *str, double *d) { int retval = getDoubleFromObject(str, d); return (retval == C_OK) ? VALKEYMODULE_OK : VALKEYMODULE_ERR; } /* Convert the string into a long double, storing it at `*ld`. * Returns VALKEYMODULE_OK on success or VALKEYMODULE_ERR if the string is * not a valid string representation of a double value. */ int VM_StringToLongDouble(const ValkeyModuleString *str, long double *ld) { int retval = string2ld(str->ptr, sdslen(str->ptr), ld); return retval ? VALKEYMODULE_OK : VALKEYMODULE_ERR; } /* Convert the string into a stream ID, storing it at `*id`. * Returns VALKEYMODULE_OK on success and returns VALKEYMODULE_ERR if the string * is not a valid string representation of a stream ID. The special IDs "+" and * "-" are allowed. */ int VM_StringToStreamID(const ValkeyModuleString *str, ValkeyModuleStreamID *id) { streamID streamid; if (streamParseID(str, &streamid) == C_OK) { id->ms = streamid.ms; id->seq = streamid.seq; return VALKEYMODULE_OK; } else { return VALKEYMODULE_ERR; } } /* Compare two string objects, returning -1, 0 or 1 respectively if * a < b, a == b, a > b. Strings are compared byte by byte as two * binary blobs without any encoding care / collation attempt. */ int VM_StringCompare(const ValkeyModuleString *a, const ValkeyModuleString *b) { return compareStringObjects(a, b); } /* Return the (possibly modified in encoding) input 'str' object if * the string is unshared, otherwise NULL is returned. */ ValkeyModuleString *moduleAssertUnsharedString(ValkeyModuleString *str) { if (str->refcount != 1) { serverLog(LL_WARNING, "Module attempted to use an in-place string modify operation " "with a string referenced multiple times. Please check the code " "for API usage correctness."); return NULL; } if (str->encoding == OBJ_ENCODING_EMBSTR) { /* Note: here we "leak" the additional allocation that was * used in order to store the embedded string in the object. */ str->ptr = sdsnewlen(str->ptr, sdslen(str->ptr)); str->encoding = OBJ_ENCODING_RAW; } else if (str->encoding == OBJ_ENCODING_INT) { /* Convert the string from integer to raw encoding. */ str->ptr = sdsfromlonglong((long)str->ptr); str->encoding = OBJ_ENCODING_RAW; } return str; } /* Append the specified buffer to the string 'str'. The string must be a * string created by the user that is referenced only a single time, otherwise * VALKEYMODULE_ERR is returned and the operation is not performed. */ int VM_StringAppendBuffer(ValkeyModuleCtx *ctx, ValkeyModuleString *str, const char *buf, size_t len) { UNUSED(ctx); str = moduleAssertUnsharedString(str); if (str == NULL) return VALKEYMODULE_ERR; str->ptr = sdscatlen(str->ptr, buf, len); return VALKEYMODULE_OK; } /* Trim possible excess memory allocated for a ValkeyModuleString. * * Sometimes a ValkeyModuleString may have more memory allocated for * it than required, typically for argv arguments that were constructed * from network buffers. This function optimizes such strings by reallocating * their memory, which is useful for strings that are not short lived but * retained for an extended duration. * * This operation is *not thread safe* and should only be called when * no concurrent access to the string is guaranteed. Using it for an argv * string in a module command before the string is potentially available * to other threads is generally safe. * * Currently, the server may also automatically trim retained strings when a * module command returns. However, doing this explicitly should still be * a preferred option: * * 1. Future versions of the server may abandon auto-trimming. * 2. Auto-trimming as currently implemented is *not thread safe*. * A background thread manipulating a recently retained string may end up * in a race condition with the auto-trim, which could result with * data corruption. */ void VM_TrimStringAllocation(ValkeyModuleString *str) { if (!str) return; trimStringObjectIfNeeded(str, 1); } /* -------------------------------------------------------------------------- * ## Reply APIs * * These functions are used for sending replies to the client. * * Most functions always return VALKEYMODULE_OK so you can use it with * 'return' in order to return from the command implementation with: * * if (... some condition ...) * return ValkeyModule_ReplyWithLongLong(ctx,mycount); * * ### Reply with collection functions * * After starting a collection reply, the module must make calls to other * `ReplyWith*` style functions in order to emit the elements of the collection. * Collection types include: Array, Map, Set and Attribute. * * When producing collections with a number of elements that is not known * beforehand, the function can be called with a special flag * VALKEYMODULE_POSTPONED_LEN (VALKEYMODULE_POSTPONED_ARRAY_LEN in the past), * and the actual number of elements can be later set with VM_ReplySet*Length() * call (which will set the latest "open" count if there are multiple ones). * -------------------------------------------------------------------------- */ /* Send an error about the number of arguments given to the command, * citing the command name in the error message. Returns VALKEYMODULE_OK. * * Example: * * if (argc != 3) return ValkeyModule_WrongArity(ctx); */ int VM_WrongArity(ValkeyModuleCtx *ctx) { addReplyErrorArity(ctx->client); return VALKEYMODULE_OK; } /* Return the client object the `VM_Reply*` functions should target. * Normally this is just `ctx->client`, that is the client that called * the module command, however in the case of thread safe contexts there * is no directly associated client (since it would not be safe to access * the client from a thread), so instead the blocked client object referenced * in the thread safe context, has a fake client that we just use to accumulate * the replies. Later, when the client is unblocked, the accumulated replies * are appended to the actual client. * * The function returns the client pointer depending on the context, or * NULL if there is no potential client. This happens when we are in the * context of a thread safe context that was not initialized with a blocked * client object. Other contexts without associated clients are the ones * initialized to run the timers callbacks. */ client *moduleGetReplyClient(ValkeyModuleCtx *ctx) { if (ctx->flags & VALKEYMODULE_CTX_THREAD_SAFE) { if (ctx->blocked_client) return ctx->blocked_client->reply_client; else return NULL; } else { /* If this is a non thread safe context, just return the client * that is running the command if any. This may be NULL as well * in the case of contexts that are not executed with associated * clients, like timer contexts. */ return ctx->client; } } /* Send an integer reply to the client, with the specified `long long` value. * The function always returns VALKEYMODULE_OK. */ int VM_ReplyWithLongLong(ValkeyModuleCtx *ctx, long long ll) { client *c = moduleGetReplyClient(ctx); if (c == NULL) return VALKEYMODULE_OK; addReplyLongLong(c, ll); return VALKEYMODULE_OK; } /* Reply with the error 'err'. * * Note that 'err' must contain all the error, including * the initial error code. The function only provides the initial "-", so * the usage is, for example: * * ValkeyModule_ReplyWithError(ctx,"ERR Wrong Type"); * * and not just: * * ValkeyModule_ReplyWithError(ctx,"Wrong Type"); * * The function always returns VALKEYMODULE_OK. */ int VM_ReplyWithError(ValkeyModuleCtx *ctx, const char *err) { client *c = moduleGetReplyClient(ctx); if (c == NULL) return VALKEYMODULE_OK; addReplyErrorFormat(c, "-%s", err); return VALKEYMODULE_OK; } /* Reply with the error create from a printf format and arguments. * * Note that 'fmt' must contain all the error, including * the initial error code. The function only provides the initial "-", so * the usage is, for example: * * ValkeyModule_ReplyWithErrorFormat(ctx,"ERR Wrong Type: %s",type); * * and not just: * * ValkeyModule_ReplyWithErrorFormat(ctx,"Wrong Type: %s",type); * * The function always returns VALKEYMODULE_OK. */ int VM_ReplyWithErrorFormat(ValkeyModuleCtx *ctx, const char *fmt, ...) { client *c = moduleGetReplyClient(ctx); if (c == NULL) return VALKEYMODULE_OK; int len = strlen(fmt) + 2; /* 1 for the \0 and 1 for the hyphen */ char *hyphenfmt = zmalloc(len); snprintf(hyphenfmt, len, "-%s", fmt); va_list ap; va_start(ap, fmt); addReplyErrorFormatInternal(c, 0, hyphenfmt, ap); va_end(ap); zfree(hyphenfmt); return VALKEYMODULE_OK; } /* Reply with a simple string (`+... \r\n` in RESP protocol). This replies * are suitable only when sending a small non-binary string with small * overhead, like "OK" or similar replies. * * The function always returns VALKEYMODULE_OK. */ int VM_ReplyWithSimpleString(ValkeyModuleCtx *ctx, const char *msg) { client *c = moduleGetReplyClient(ctx); if (c == NULL) return VALKEYMODULE_OK; addReplyProto(c, "+", 1); addReplyProto(c, msg, strlen(msg)); addReplyProto(c, "\r\n", 2); return VALKEYMODULE_OK; } #define COLLECTION_REPLY_ARRAY 1 #define COLLECTION_REPLY_MAP 2 #define COLLECTION_REPLY_SET 3 #define COLLECTION_REPLY_ATTRIBUTE 4 int moduleReplyWithCollection(ValkeyModuleCtx *ctx, long len, int type) { client *c = moduleGetReplyClient(ctx); if (c == NULL) return VALKEYMODULE_OK; if (len == VALKEYMODULE_POSTPONED_LEN) { ctx->postponed_arrays = zrealloc(ctx->postponed_arrays, sizeof(void *) * (ctx->postponed_arrays_count + 1)); ctx->postponed_arrays[ctx->postponed_arrays_count] = addReplyDeferredLen(c); ctx->postponed_arrays_count++; } else if (len == 0) { switch (type) { case COLLECTION_REPLY_ARRAY: addReply(c, shared.emptyarray); break; case COLLECTION_REPLY_MAP: addReply(c, shared.emptymap[c->resp]); break; case COLLECTION_REPLY_SET: addReply(c, shared.emptyset[c->resp]); break; case COLLECTION_REPLY_ATTRIBUTE: addReplyAttributeLen(c, len); break; default: serverPanic("Invalid module empty reply type %d", type); } } else { switch (type) { case COLLECTION_REPLY_ARRAY: addReplyArrayLen(c, len); break; case COLLECTION_REPLY_MAP: addReplyMapLen(c, len); break; case COLLECTION_REPLY_SET: addReplySetLen(c, len); break; case COLLECTION_REPLY_ATTRIBUTE: addReplyAttributeLen(c, len); break; default: serverPanic("Invalid module reply type %d", type); } } return VALKEYMODULE_OK; } /* Reply with an array type of 'len' elements. * * After starting an array reply, the module must make `len` calls to other * `ReplyWith*` style functions in order to emit the elements of the array. * See Reply APIs section for more details. * * Use VM_ReplySetArrayLength() to set deferred length. * * The function always returns VALKEYMODULE_OK. */ int VM_ReplyWithArray(ValkeyModuleCtx *ctx, long len) { return moduleReplyWithCollection(ctx, len, COLLECTION_REPLY_ARRAY); } /* Reply with a RESP3 Map type of 'len' pairs. * Visit https://valkey.io/topics/protocol for more info about RESP3. * * After starting a map reply, the module must make `len*2` calls to other * `ReplyWith*` style functions in order to emit the elements of the map. * See Reply APIs section for more details. * * If the connected client is using RESP2, the reply will be converted to a flat * array. * * Use VM_ReplySetMapLength() to set deferred length. * * The function always returns VALKEYMODULE_OK. */ int VM_ReplyWithMap(ValkeyModuleCtx *ctx, long len) { return moduleReplyWithCollection(ctx, len, COLLECTION_REPLY_MAP); } /* Reply with a RESP3 Set type of 'len' elements. * Visit https://valkey.io/topics/protocol for more info about RESP3. * * After starting a set reply, the module must make `len` calls to other * `ReplyWith*` style functions in order to emit the elements of the set. * See Reply APIs section for more details. * * If the connected client is using RESP2, the reply will be converted to an * array type. * * Use VM_ReplySetSetLength() to set deferred length. * * The function always returns VALKEYMODULE_OK. */ int VM_ReplyWithSet(ValkeyModuleCtx *ctx, long len) { return moduleReplyWithCollection(ctx, len, COLLECTION_REPLY_SET); } /* Add attributes (metadata) to the reply. Should be done before adding the * actual reply. see https://valkey.io/topics/protocol#attribute-type * * After starting an attribute's reply, the module must make `len*2` calls to other * `ReplyWith*` style functions in order to emit the elements of the attribute map. * See Reply APIs section for more details. * * Use VM_ReplySetAttributeLength() to set deferred length. * * Not supported by RESP2 and will return VALKEYMODULE_ERR, otherwise * the function always returns VALKEYMODULE_OK. */ int VM_ReplyWithAttribute(ValkeyModuleCtx *ctx, long len) { if (ctx->client->resp == 2) return VALKEYMODULE_ERR; return moduleReplyWithCollection(ctx, len, COLLECTION_REPLY_ATTRIBUTE); } /* Reply to the client with a null array, simply null in RESP3, * null array in RESP2. * * Note: In RESP3 there's no difference between Null reply and * NullArray reply, so to prevent ambiguity it's better to avoid * using this API and use ValkeyModule_ReplyWithNull instead. * * The function always returns VALKEYMODULE_OK. */ int VM_ReplyWithNullArray(ValkeyModuleCtx *ctx) { client *c = moduleGetReplyClient(ctx); if (c == NULL) return VALKEYMODULE_OK; addReplyNullArray(c); return VALKEYMODULE_OK; } /* Reply to the client with an empty array. * * The function always returns VALKEYMODULE_OK. */ int VM_ReplyWithEmptyArray(ValkeyModuleCtx *ctx) { client *c = moduleGetReplyClient(ctx); if (c == NULL) return VALKEYMODULE_OK; addReply(c, shared.emptyarray); return VALKEYMODULE_OK; } void moduleReplySetCollectionLength(ValkeyModuleCtx *ctx, long len, int type) { client *c = moduleGetReplyClient(ctx); if (c == NULL) return; if (ctx->postponed_arrays_count == 0) { serverLog(LL_WARNING, "API misuse detected in module %s: " "ValkeyModule_ReplySet*Length() called without previous " "ValkeyModule_ReplyWith*(ctx,VALKEYMODULE_POSTPONED_LEN) " "call.", ctx->module->name); return; } ctx->postponed_arrays_count--; switch (type) { case COLLECTION_REPLY_ARRAY: setDeferredArrayLen(c, ctx->postponed_arrays[ctx->postponed_arrays_count], len); break; case COLLECTION_REPLY_MAP: setDeferredMapLen(c, ctx->postponed_arrays[ctx->postponed_arrays_count], len); break; case COLLECTION_REPLY_SET: setDeferredSetLen(c, ctx->postponed_arrays[ctx->postponed_arrays_count], len); break; case COLLECTION_REPLY_ATTRIBUTE: setDeferredAttributeLen(c, ctx->postponed_arrays[ctx->postponed_arrays_count], len); break; default: serverPanic("Invalid module reply type %d", type); } if (ctx->postponed_arrays_count == 0) { zfree(ctx->postponed_arrays); ctx->postponed_arrays = NULL; } } /* When ValkeyModule_ReplyWithArray() is used with the argument * VALKEYMODULE_POSTPONED_LEN, because we don't know beforehand the number * of items we are going to output as elements of the array, this function * will take care to set the array length. * * Since it is possible to have multiple array replies pending with unknown * length, this function guarantees to always set the latest array length * that was created in a postponed way. * * For example in order to output an array like [1,[10,20,30]] we * could write: * * ValkeyModule_ReplyWithArray(ctx,VALKEYMODULE_POSTPONED_LEN); * ValkeyModule_ReplyWithLongLong(ctx,1); * ValkeyModule_ReplyWithArray(ctx,VALKEYMODULE_POSTPONED_LEN); * ValkeyModule_ReplyWithLongLong(ctx,10); * ValkeyModule_ReplyWithLongLong(ctx,20); * ValkeyModule_ReplyWithLongLong(ctx,30); * ValkeyModule_ReplySetArrayLength(ctx,3); // Set len of 10,20,30 array. * ValkeyModule_ReplySetArrayLength(ctx,2); // Set len of top array * * Note that in the above example there is no reason to postpone the array * length, since we produce a fixed number of elements, but in the practice * the code may use an iterator or other ways of creating the output so * that is not easy to calculate in advance the number of elements. */ void VM_ReplySetArrayLength(ValkeyModuleCtx *ctx, long len) { moduleReplySetCollectionLength(ctx, len, COLLECTION_REPLY_ARRAY); } /* Very similar to ValkeyModule_ReplySetArrayLength except `len` should * exactly half of the number of `ReplyWith*` functions called in the * context of the map. * Visit https://valkey.io/topics/protocol for more info about RESP3. */ void VM_ReplySetMapLength(ValkeyModuleCtx *ctx, long len) { moduleReplySetCollectionLength(ctx, len, COLLECTION_REPLY_MAP); } /* Very similar to ValkeyModule_ReplySetArrayLength * Visit https://valkey.io/topics/protocol for more info about RESP3. */ void VM_ReplySetSetLength(ValkeyModuleCtx *ctx, long len) { moduleReplySetCollectionLength(ctx, len, COLLECTION_REPLY_SET); } /* Very similar to ValkeyModule_ReplySetMapLength * Visit https://valkey.io/topics/protocol for more info about RESP3. * * Must not be called if VM_ReplyWithAttribute returned an error. */ void VM_ReplySetAttributeLength(ValkeyModuleCtx *ctx, long len) { if (ctx->client->resp == 2) return; moduleReplySetCollectionLength(ctx, len, COLLECTION_REPLY_ATTRIBUTE); } /* Reply with a bulk string, taking in input a C buffer pointer and length. * * The function always returns VALKEYMODULE_OK. */ int VM_ReplyWithStringBuffer(ValkeyModuleCtx *ctx, const char *buf, size_t len) { client *c = moduleGetReplyClient(ctx); if (c == NULL) return VALKEYMODULE_OK; addReplyBulkCBuffer(c, (char *)buf, len); return VALKEYMODULE_OK; } /* Reply with a bulk string, taking in input a C buffer pointer that is * assumed to be null-terminated. * * The function always returns VALKEYMODULE_OK. */ int VM_ReplyWithCString(ValkeyModuleCtx *ctx, const char *buf) { client *c = moduleGetReplyClient(ctx); if (c == NULL) return VALKEYMODULE_OK; addReplyBulkCString(c, (char *)buf); return VALKEYMODULE_OK; } /* Reply with a bulk string, taking in input a ValkeyModuleString object. * * The function always returns VALKEYMODULE_OK. */ int VM_ReplyWithString(ValkeyModuleCtx *ctx, ValkeyModuleString *str) { client *c = moduleGetReplyClient(ctx); if (c == NULL) return VALKEYMODULE_OK; addReplyBulk(c, str); return VALKEYMODULE_OK; } /* Reply with an empty string. * * The function always returns VALKEYMODULE_OK. */ int VM_ReplyWithEmptyString(ValkeyModuleCtx *ctx) { client *c = moduleGetReplyClient(ctx); if (c == NULL) return VALKEYMODULE_OK; addReply(c, shared.emptybulk); return VALKEYMODULE_OK; } /* Reply with a binary safe string, which should not be escaped or filtered * taking in input a C buffer pointer, length and a 3 character type/extension. * * The function always returns VALKEYMODULE_OK. */ int VM_ReplyWithVerbatimStringType(ValkeyModuleCtx *ctx, const char *buf, size_t len, const char *ext) { client *c = moduleGetReplyClient(ctx); if (c == NULL) return VALKEYMODULE_OK; addReplyVerbatim(c, buf, len, ext); return VALKEYMODULE_OK; } /* Reply with a binary safe string, which should not be escaped or filtered * taking in input a C buffer pointer and length. * * The function always returns VALKEYMODULE_OK. */ int VM_ReplyWithVerbatimString(ValkeyModuleCtx *ctx, const char *buf, size_t len) { return VM_ReplyWithVerbatimStringType(ctx, buf, len, "txt"); } /* Reply to the client with a NULL. * * The function always returns VALKEYMODULE_OK. */ int VM_ReplyWithNull(ValkeyModuleCtx *ctx) { client *c = moduleGetReplyClient(ctx); if (c == NULL) return VALKEYMODULE_OK; addReplyNull(c); return VALKEYMODULE_OK; } /* Reply with a RESP3 Boolean type. * Visit https://valkey.io/topics/protocol for more info about RESP3. * * In RESP3, this is boolean type * In RESP2, it's a string response of "1" and "0" for true and false respectively. * * The function always returns VALKEYMODULE_OK. */ int VM_ReplyWithBool(ValkeyModuleCtx *ctx, int b) { client *c = moduleGetReplyClient(ctx); if (c == NULL) return VALKEYMODULE_OK; addReplyBool(c, b); return VALKEYMODULE_OK; } /* Reply exactly what a command returned us with ValkeyModule_Call(). * This function is useful when we use ValkeyModule_Call() in order to * execute some command, as we want to reply to the client exactly the * same reply we obtained by the command. * * Return: * - VALKEYMODULE_OK on success. * - VALKEYMODULE_ERR if the given reply is in RESP3 format but the client expects RESP2. * In case of an error, it's the module writer responsibility to translate the reply * to RESP2 (or handle it differently by returning an error). Notice that for * module writer convenience, it is possible to pass `0` as a parameter to the fmt * argument of `VM_Call` so that the ValkeyModuleCallReply will return in the same * protocol (RESP2 or RESP3) as set in the current client's context. */ int VM_ReplyWithCallReply(ValkeyModuleCtx *ctx, ValkeyModuleCallReply *reply) { client *c = moduleGetReplyClient(ctx); if (c == NULL) return VALKEYMODULE_OK; if (c->resp == 2 && callReplyIsResp3(reply)) { /* The reply is in RESP3 format and the client is RESP2, * so it isn't possible to send this reply to the client. */ return VALKEYMODULE_ERR; } size_t proto_len; const char *proto = callReplyGetProto(reply, &proto_len); addReplyProto(c, proto, proto_len); /* Propagate the error list from that reply to the other client, to do some * post error reply handling, like statistics. * Note that if the original reply had an array with errors, and the module * replied with just a portion of the original reply, and not the entire * reply, the errors are currently not propagated and the errors stats * will not get propagated. */ list *errors = callReplyDeferredErrorList(reply); if (errors) deferredAfterErrorReply(c, errors); return VALKEYMODULE_OK; } /* Reply with a RESP3 Double type. * Visit https://valkey.io/topics/protocol for more info about RESP3. * * Send a string reply obtained converting the double 'd' into a bulk string. * This function is basically equivalent to converting a double into * a string into a C buffer, and then calling the function * ValkeyModule_ReplyWithStringBuffer() with the buffer and length. * * In RESP3 the string is tagged as a double, while in RESP2 it's just a plain string * that the user will have to parse. * * The function always returns VALKEYMODULE_OK. */ int VM_ReplyWithDouble(ValkeyModuleCtx *ctx, double d) { client *c = moduleGetReplyClient(ctx); if (c == NULL) return VALKEYMODULE_OK; addReplyDouble(c, d); return VALKEYMODULE_OK; } /* Reply with a RESP3 BigNumber type. * Visit https://valkey.io/topics/protocol for more info about RESP3. * * In RESP3, this is a string of length `len` that is tagged as a BigNumber, * however, it's up to the caller to ensure that it's a valid BigNumber. * In RESP2, this is just a plain bulk string response. * * The function always returns VALKEYMODULE_OK. */ int VM_ReplyWithBigNumber(ValkeyModuleCtx *ctx, const char *bignum, size_t len) { client *c = moduleGetReplyClient(ctx); if (c == NULL) return VALKEYMODULE_OK; addReplyBigNum(c, bignum, len); return VALKEYMODULE_OK; } /* Send a string reply obtained converting the long double 'ld' into a bulk * string. This function is basically equivalent to converting a long double * into a string into a C buffer, and then calling the function * ValkeyModule_ReplyWithStringBuffer() with the buffer and length. * The double string uses human readable formatting (see * `addReplyHumanLongDouble` in networking.c). * * The function always returns VALKEYMODULE_OK. */ int VM_ReplyWithLongDouble(ValkeyModuleCtx *ctx, long double ld) { client *c = moduleGetReplyClient(ctx); if (c == NULL) return VALKEYMODULE_OK; addReplyHumanLongDouble(c, ld); return VALKEYMODULE_OK; } /* -------------------------------------------------------------------------- * ## Commands replication API * -------------------------------------------------------------------------- */ /* Replicate the specified command and arguments to replicas and AOF, as effect * of execution of the calling command implementation. * * The replicated commands are always wrapped into the MULTI/EXEC that * contains all the commands replicated in a given module command * execution. However the commands replicated with ValkeyModule_Call() * are the first items, the ones replicated with ValkeyModule_Replicate() * will all follow before the EXEC. * * Modules should try to use one interface or the other. * * This command follows exactly the same interface of ValkeyModule_Call(), * so a set of format specifiers must be passed, followed by arguments * matching the provided format specifiers. * * Please refer to ValkeyModule_Call() for more information. * * Using the special "A" and "R" modifiers, the caller can exclude either * the AOF or the replicas from the propagation of the specified command. * Otherwise, by default, the command will be propagated in both channels. * * #### Note about calling this function from a thread safe context: * * Normally when you call this function from the callback implementing a * module command, or any other callback provided by the Module API, * The server will accumulate all the calls to this function in the context of * the callback, and will propagate all the commands wrapped in a MULTI/EXEC * transaction. However when calling this function from a threaded safe context * that can live an undefined amount of time, and can be locked/unlocked in * at will, the behavior is different: MULTI/EXEC wrapper is not emitted * and the command specified is inserted in the AOF and replication stream * immediately. * * #### Return value * * The command returns VALKEYMODULE_ERR if the format specifiers are invalid * or the command name does not belong to a known command. */ int VM_Replicate(ValkeyModuleCtx *ctx, const char *cmdname, const char *fmt, ...) { struct serverCommand *cmd; robj **argv = NULL; int argc = 0, flags = 0, j; va_list ap; cmd = lookupCommandByCString((char *)cmdname); if (!cmd) return VALKEYMODULE_ERR; /* Create the client and dispatch the command. */ va_start(ap, fmt); argv = moduleCreateArgvFromUserFormat(cmdname, fmt, &argc, &flags, ap); va_end(ap); if (argv == NULL) return VALKEYMODULE_ERR; /* Select the propagation target. Usually is AOF + replicas, however * the caller can exclude one or the other using the "A" or "R" * modifiers. */ int target = 0; if (!(flags & VALKEYMODULE_ARGV_NO_AOF)) target |= PROPAGATE_AOF; if (!(flags & VALKEYMODULE_ARGV_NO_REPLICAS)) target |= PROPAGATE_REPL; alsoPropagate(ctx->client->db->id, argv, argc, target); /* Release the argv. */ for (j = 0; j < argc; j++) decrRefCount(argv[j]); zfree(argv); server.dirty++; return VALKEYMODULE_OK; } /* This function will replicate the command exactly as it was invoked * by the client. Note that this function will not wrap the command into * a MULTI/EXEC stanza, so it should not be mixed with other replication * commands. * * Basically this form of replication is useful when you want to propagate * the command to the replicas and AOF file exactly as it was called, since * the command can just be re-executed to deterministically re-create the * new state starting from the old one. * * The function always returns VALKEYMODULE_OK. */ int VM_ReplicateVerbatim(ValkeyModuleCtx *ctx) { alsoPropagate(ctx->client->db->id, ctx->client->argv, ctx->client->argc, PROPAGATE_AOF | PROPAGATE_REPL); server.dirty++; return VALKEYMODULE_OK; } /* -------------------------------------------------------------------------- * ## DB and Key APIs -- Generic API * -------------------------------------------------------------------------- */ /* Return the ID of the current client calling the currently active module * command. The returned ID has a few guarantees: * * 1. The ID is different for each different client, so if the same client * executes a module command multiple times, it can be recognized as * having the same ID, otherwise the ID will be different. * 2. The ID increases monotonically. Clients connecting to the server later * are guaranteed to get IDs greater than any past ID previously seen. * * Valid IDs are from 1 to 2^64 - 1. If 0 is returned it means there is no way * to fetch the ID in the context the function was currently called. * * After obtaining the ID, it is possible to check if the command execution * is actually happening in the context of AOF loading, using this macro: * * if (ValkeyModule_IsAOFClient(ValkeyModule_GetClientId(ctx)) { * // Handle it differently. * } */ unsigned long long VM_GetClientId(ValkeyModuleCtx *ctx) { if (ctx->client == NULL) return 0; return ctx->client->id; } /* Return the ACL user name used by the client with the specified client ID. * Client ID can be obtained with VM_GetClientId() API. If the client does not * exist, NULL is returned and errno is set to ENOENT. If the client isn't * using an ACL user, NULL is returned and errno is set to ENOTSUP */ ValkeyModuleString *VM_GetClientUserNameById(ValkeyModuleCtx *ctx, uint64_t id) { client *client = lookupClientByID(id); if (client == NULL) { errno = ENOENT; return NULL; } if (client->user == NULL) { errno = ENOTSUP; return NULL; } sds name = sdsnew(client->user->name); robj *str = createObject(OBJ_STRING, name); autoMemoryAdd(ctx, VALKEYMODULE_AM_STRING, str); return str; } /* This is a helper for VM_GetClientInfoById() and other functions: given * a client, it populates the client info structure with the appropriate * fields depending on the version provided. If the version is not valid * then VALKEYMODULE_ERR is returned. Otherwise the function returns * VALKEYMODULE_OK and the structure pointed by 'ci' gets populated. */ int modulePopulateClientInfoStructure(void *ci, client *client, int structver) { if (structver != 1) return VALKEYMODULE_ERR; ValkeyModuleClientInfoV1 *ci1 = ci; memset(ci1, 0, sizeof(*ci1)); ci1->version = structver; if (client->flag.multi) ci1->flags |= VALKEYMODULE_CLIENTINFO_FLAG_MULTI; if (client->flag.pubsub) ci1->flags |= VALKEYMODULE_CLIENTINFO_FLAG_PUBSUB; if (client->flag.unix_socket) ci1->flags |= VALKEYMODULE_CLIENTINFO_FLAG_UNIXSOCKET; if (client->flag.tracking) ci1->flags |= VALKEYMODULE_CLIENTINFO_FLAG_TRACKING; if (client->flag.blocked) ci1->flags |= VALKEYMODULE_CLIENTINFO_FLAG_BLOCKED; if (client->conn->type == connectionTypeTls()) ci1->flags |= VALKEYMODULE_CLIENTINFO_FLAG_SSL; int port; connAddrPeerName(client->conn, ci1->addr, sizeof(ci1->addr), &port); ci1->port = port; ci1->db = client->db->id; ci1->id = client->id; return VALKEYMODULE_OK; } /* This is a helper for moduleFireServerEvent() and other functions: * It populates the replication info structure with the appropriate * fields depending on the version provided. If the version is not valid * then VALKEYMODULE_ERR is returned. Otherwise the function returns * VALKEYMODULE_OK and the structure pointed by 'ri' gets populated. */ int modulePopulateReplicationInfoStructure(void *ri, int structver) { if (structver != 1) return VALKEYMODULE_ERR; ValkeyModuleReplicationInfoV1 *ri1 = ri; memset(ri1, 0, sizeof(*ri1)); ri1->version = structver; ri1->primary = server.primary_host == NULL; ri1->primary_host = server.primary_host ? server.primary_host : ""; ri1->primary_port = server.primary_port; ri1->replid1 = server.replid; ri1->replid2 = server.replid2; ri1->repl1_offset = server.primary_repl_offset; ri1->repl2_offset = server.second_replid_offset; return VALKEYMODULE_OK; } /* Return information about the client with the specified ID (that was * previously obtained via the ValkeyModule_GetClientId() API). If the * client exists, VALKEYMODULE_OK is returned, otherwise VALKEYMODULE_ERR * is returned. * * When the client exist and the `ci` pointer is not NULL, but points to * a structure of type ValkeyModuleClientInfoV1, previously initialized with * the correct VALKEYMODULE_CLIENTINFO_INITIALIZER_V1, the structure is populated * with the following fields: * * uint64_t flags; // VALKEYMODULE_CLIENTINFO_FLAG_* * uint64_t id; // Client ID * char addr[46]; // IPv4 or IPv6 address. * uint16_t port; // TCP port. * uint16_t db; // Selected DB. * * Note: the client ID is useless in the context of this call, since we * already know, however the same structure could be used in other * contexts where we don't know the client ID, yet the same structure * is returned. * * With flags having the following meaning: * * VALKEYMODULE_CLIENTINFO_FLAG_SSL Client using SSL connection. * VALKEYMODULE_CLIENTINFO_FLAG_PUBSUB Client in Pub/Sub mode. * VALKEYMODULE_CLIENTINFO_FLAG_BLOCKED Client blocked in command. * VALKEYMODULE_CLIENTINFO_FLAG_TRACKING Client with keys tracking on. * VALKEYMODULE_CLIENTINFO_FLAG_UNIXSOCKET Client using unix domain socket. * VALKEYMODULE_CLIENTINFO_FLAG_MULTI Client in MULTI state. * * However passing NULL is a way to just check if the client exists in case * we are not interested in any additional information. * * This is the correct usage when we want the client info structure * returned: * * ValkeyModuleClientInfo ci = VALKEYMODULE_CLIENTINFO_INITIALIZER; * int retval = ValkeyModule_GetClientInfoById(&ci,client_id); * if (retval == VALKEYMODULE_OK) { * printf("Address: %s\n", ci.addr); * } */ int VM_GetClientInfoById(void *ci, uint64_t id) { client *client = lookupClientByID(id); if (client == NULL) return VALKEYMODULE_ERR; if (ci == NULL) return VALKEYMODULE_OK; /* Fill the info structure if passed. */ uint64_t structver = ((uint64_t *)ci)[0]; return modulePopulateClientInfoStructure(ci, client, structver); } /* Returns the name of the client connection with the given ID. * * If the client ID does not exist or if the client has no name associated with * it, NULL is returned. */ ValkeyModuleString *VM_GetClientNameById(ValkeyModuleCtx *ctx, uint64_t id) { client *client = lookupClientByID(id); if (client == NULL || client->name == NULL) return NULL; robj *name = client->name; incrRefCount(name); autoMemoryAdd(ctx, VALKEYMODULE_AM_STRING, name); return name; } /* Sets the name of the client with the given ID. This is equivalent to the client calling * `CLIENT SETNAME name`. * * Returns VALKEYMODULE_OK on success. On failure, VALKEYMODULE_ERR is returned * and errno is set as follows: * * - ENOENT if the client does not exist * - EINVAL if the name contains invalid characters */ int VM_SetClientNameById(uint64_t id, ValkeyModuleString *name) { client *client = lookupClientByID(id); if (client == NULL) { errno = ENOENT; return VALKEYMODULE_ERR; } if (clientSetName(client, name, NULL) == C_ERR) { errno = EINVAL; return VALKEYMODULE_ERR; } return VALKEYMODULE_OK; } /* Publish a message to subscribers (see PUBLISH command). */ int VM_PublishMessage(ValkeyModuleCtx *ctx, ValkeyModuleString *channel, ValkeyModuleString *message) { UNUSED(ctx); return pubsubPublishMessageAndPropagateToCluster(channel, message, 0); } /* Publish a message to shard-subscribers (see SPUBLISH command). */ int VM_PublishMessageShard(ValkeyModuleCtx *ctx, ValkeyModuleString *channel, ValkeyModuleString *message) { UNUSED(ctx); return pubsubPublishMessageAndPropagateToCluster(channel, message, 1); } /* Return the currently selected DB. */ int VM_GetSelectedDb(ValkeyModuleCtx *ctx) { return ctx->client->db->id; } /* Return the current context's flags. The flags provide information on the * current request context (whether the client is a Lua script or in a MULTI), * and about the instance in general, i.e replication and persistence. * * It is possible to call this function even with a NULL context, however * in this case the following flags will not be reported: * * * LUA, MULTI, REPLICATED, DIRTY (see below for more info). * * Available flags and their meaning: * * * VALKEYMODULE_CTX_FLAGS_LUA: The command is running in a Lua script * * * VALKEYMODULE_CTX_FLAGS_MULTI: The command is running inside a transaction * * * VALKEYMODULE_CTX_FLAGS_REPLICATED: The command was sent over the replication * link by the PRIMARY * * * VALKEYMODULE_CTX_FLAGS_PRIMARY: The instance is a primary * * * VALKEYMODULE_CTX_FLAGS_REPLICA: The instance is a replica * * * VALKEYMODULE_CTX_FLAGS_READONLY: The instance is read-only * * * VALKEYMODULE_CTX_FLAGS_CLUSTER: The instance is in cluster mode * * * VALKEYMODULE_CTX_FLAGS_AOF: The instance has AOF enabled * * * VALKEYMODULE_CTX_FLAGS_RDB: The instance has RDB enabled * * * VALKEYMODULE_CTX_FLAGS_MAXMEMORY: The instance has Maxmemory set * * * VALKEYMODULE_CTX_FLAGS_EVICT: Maxmemory is set and has an eviction * policy that may delete keys * * * VALKEYMODULE_CTX_FLAGS_OOM: The server is out of memory according to the * maxmemory setting. * * * VALKEYMODULE_CTX_FLAGS_OOM_WARNING: Less than 25% of memory remains before * reaching the maxmemory level. * * * VALKEYMODULE_CTX_FLAGS_LOADING: Server is loading RDB/AOF * * * VALKEYMODULE_CTX_FLAGS_REPLICA_IS_STALE: No active link with the primary. * * * VALKEYMODULE_CTX_FLAGS_REPLICA_IS_CONNECTING: The replica is trying to * connect with the primary. * * * VALKEYMODULE_CTX_FLAGS_REPLICA_IS_TRANSFERRING: primary -> Replica RDB * transfer is in progress. * * * VALKEYMODULE_CTX_FLAGS_REPLICA_IS_ONLINE: The replica has an active link * with its primary. This is the * contrary of STALE state. * * * VALKEYMODULE_CTX_FLAGS_ACTIVE_CHILD: There is currently some background * process active (RDB, AUX or module). * * * VALKEYMODULE_CTX_FLAGS_MULTI_DIRTY: The next EXEC will fail due to dirty * CAS (touched keys). * * * VALKEYMODULE_CTX_FLAGS_IS_CHILD: The server is currently running inside * background child process. * * * VALKEYMODULE_CTX_FLAGS_RESP3: Indicate the that client attached to this * context is using RESP3. * * * VALKEYMODULE_CTX_FLAGS_SERVER_STARTUP: The instance is starting */ int VM_GetContextFlags(ValkeyModuleCtx *ctx) { int flags = 0; /* Client specific flags */ if (ctx) { if (ctx->client) { if (ctx->client->flag.deny_blocking) flags |= VALKEYMODULE_CTX_FLAGS_DENY_BLOCKING; /* Module command received from PRIMARY, is replicated. */ if (ctx->client->flag.primary) flags |= VALKEYMODULE_CTX_FLAGS_REPLICATED; if (ctx->client->resp == 3) { flags |= VALKEYMODULE_CTX_FLAGS_RESP3; } } /* For DIRTY flags, we need the blocked client if used */ client *c = ctx->blocked_client ? ctx->blocked_client->client : ctx->client; if (c && (c->flag.dirty_cas || c->flag.dirty_exec)) { flags |= VALKEYMODULE_CTX_FLAGS_MULTI_DIRTY; } } if (scriptIsRunning()) flags |= VALKEYMODULE_CTX_FLAGS_LUA; if (server.in_exec) flags |= VALKEYMODULE_CTX_FLAGS_MULTI; if (server.cluster_enabled) flags |= VALKEYMODULE_CTX_FLAGS_CLUSTER; if (server.async_loading) flags |= VALKEYMODULE_CTX_FLAGS_ASYNC_LOADING; else if (server.loading) flags |= VALKEYMODULE_CTX_FLAGS_LOADING; /* Maxmemory and eviction policy */ if (server.maxmemory > 0 && (!server.primary_host || !server.repl_replica_ignore_maxmemory)) { flags |= VALKEYMODULE_CTX_FLAGS_MAXMEMORY; if (server.maxmemory_policy != MAXMEMORY_NO_EVICTION) flags |= VALKEYMODULE_CTX_FLAGS_EVICT; } /* Persistence flags */ if (server.aof_state != AOF_OFF) flags |= VALKEYMODULE_CTX_FLAGS_AOF; if (server.saveparamslen > 0) flags |= VALKEYMODULE_CTX_FLAGS_RDB; /* Replication flags */ if (server.primary_host == NULL) { flags |= VALKEYMODULE_CTX_FLAGS_PRIMARY; } else { flags |= VALKEYMODULE_CTX_FLAGS_REPLICA; if (server.repl_replica_ro) flags |= VALKEYMODULE_CTX_FLAGS_READONLY; /* Replica state flags. */ if (server.repl_state == REPL_STATE_CONNECT || server.repl_state == REPL_STATE_CONNECTING) { flags |= VALKEYMODULE_CTX_FLAGS_REPLICA_IS_CONNECTING; } else if (server.repl_state == REPL_STATE_TRANSFER) { flags |= VALKEYMODULE_CTX_FLAGS_REPLICA_IS_TRANSFERRING; } else if (server.repl_state == REPL_STATE_CONNECTED) { flags |= VALKEYMODULE_CTX_FLAGS_REPLICA_IS_ONLINE; } if (server.repl_state != REPL_STATE_CONNECTED) flags |= VALKEYMODULE_CTX_FLAGS_REPLICA_IS_STALE; } /* OOM flag. */ float level; int retval = getMaxmemoryState(NULL, NULL, NULL, &level); if (retval == C_ERR) flags |= VALKEYMODULE_CTX_FLAGS_OOM; if (level > 0.75) flags |= VALKEYMODULE_CTX_FLAGS_OOM_WARNING; /* Presence of children processes. */ if (hasActiveChildProcess()) flags |= VALKEYMODULE_CTX_FLAGS_ACTIVE_CHILD; if (server.in_fork_child) flags |= VALKEYMODULE_CTX_FLAGS_IS_CHILD; /* Non-empty server.loadmodule_queue means that the server is starting. */ if (listLength(server.loadmodule_queue) > 0) flags |= VALKEYMODULE_CTX_FLAGS_SERVER_STARTUP; return flags; } /* Returns true if a client sent the CLIENT PAUSE command to the server or * if the Cluster does a manual failover, pausing the clients. * This is needed when we have a primary with replicas, and want to write, * without adding further data to the replication channel, that the replicas * replication offset, match the one of the primary. When this happens, it is * safe to failover the primary without data loss. * * However modules may generate traffic by calling ValkeyModule_Call() with * the "!" flag, or by calling ValkeyModule_Replicate(), in a context outside * commands execution, for instance in timeout callbacks, threads safe * contexts, and so forth. When modules will generate too much traffic, it * will be hard for the primary and replicas offset to match, because there * is more data to send in the replication channel. * * So modules may want to try to avoid very heavy background work that has * the effect of creating data to the replication channel, when this function * returns true. This is mostly useful for modules that have background * garbage collection tasks, or that do writes and replicate such writes * periodically in timer callbacks or other periodic callbacks. */ int VM_AvoidReplicaTraffic(void) { return !!(isPausedActionsWithUpdate(PAUSE_ACTION_REPLICA)); } /* Change the currently selected DB. Returns an error if the id * is out of range. * * Note that the client will retain the currently selected DB even after * the command implemented by the module calling this function * returns. * * If the module command wishes to change something in a different DB and * returns back to the original one, it should call ValkeyModule_GetSelectedDb() * before in order to restore the old DB number before returning. */ int VM_SelectDb(ValkeyModuleCtx *ctx, int newid) { int retval = selectDb(ctx->client, newid); return (retval == C_OK) ? VALKEYMODULE_OK : VALKEYMODULE_ERR; } /* Check if a key exists, without affecting its last access time. * * This is equivalent to calling VM_OpenKey with the mode VALKEYMODULE_READ | * VALKEYMODULE_OPEN_KEY_NOTOUCH, then checking if NULL was returned and, if not, * calling VM_CloseKey on the opened key. */ int VM_KeyExists(ValkeyModuleCtx *ctx, robj *keyname) { robj *value = lookupKeyReadWithFlags(ctx->client->db, keyname, LOOKUP_NOTOUCH); return (value != NULL); } /* Initialize a ValkeyModuleKey struct */ static void moduleInitKey(ValkeyModuleKey *kp, ValkeyModuleCtx *ctx, robj *keyname, robj *value, int mode) { kp->ctx = ctx; kp->db = ctx->client->db; kp->key = keyname; incrRefCount(keyname); kp->value = value; kp->iter = NULL; kp->mode = mode; if (kp->value) moduleInitKeyTypeSpecific(kp); } /* Initialize the type-specific part of the key. Only when key has a value. */ static void moduleInitKeyTypeSpecific(ValkeyModuleKey *key) { switch (key->value->type) { case OBJ_ZSET: zsetKeyReset(key); break; case OBJ_STREAM: key->u.stream.signalready = 0; break; } } /* Return a handle representing a key, so that it is possible * to call other APIs with the key handle as argument to perform * operations on the key. * * The return value is the handle representing the key, that must be * closed with VM_CloseKey(). * * If the key does not exist and VALKEYMODULE_WRITE mode is requested, the handle * is still returned, since it is possible to perform operations on * a yet not existing key (that will be created, for example, after * a list push operation). If the mode is just VALKEYMODULE_READ instead, and the * key does not exist, NULL is returned. However it is still safe to * call ValkeyModule_CloseKey() and ValkeyModule_KeyType() on a NULL * value. * * Extra flags that can be pass to the API under the mode argument: * * VALKEYMODULE_OPEN_KEY_NOTOUCH - Avoid touching the LRU/LFU of the key when opened. * * VALKEYMODULE_OPEN_KEY_NONOTIFY - Don't trigger keyspace event on key misses. * * VALKEYMODULE_OPEN_KEY_NOSTATS - Don't update keyspace hits/misses counters. * * VALKEYMODULE_OPEN_KEY_NOEXPIRE - Avoid deleting lazy expired keys. * * VALKEYMODULE_OPEN_KEY_NOEFFECTS - Avoid any effects from fetching the key. */ ValkeyModuleKey *VM_OpenKey(ValkeyModuleCtx *ctx, robj *keyname, int mode) { ValkeyModuleKey *kp; robj *value; int flags = 0; flags |= (mode & VALKEYMODULE_OPEN_KEY_NOTOUCH ? LOOKUP_NOTOUCH : 0); flags |= (mode & VALKEYMODULE_OPEN_KEY_NONOTIFY ? LOOKUP_NONOTIFY : 0); flags |= (mode & VALKEYMODULE_OPEN_KEY_NOSTATS ? LOOKUP_NOSTATS : 0); flags |= (mode & VALKEYMODULE_OPEN_KEY_NOEXPIRE ? LOOKUP_NOEXPIRE : 0); flags |= (mode & VALKEYMODULE_OPEN_KEY_NOEFFECTS ? LOOKUP_NOEFFECTS : 0); if (mode & VALKEYMODULE_WRITE) { value = lookupKeyWriteWithFlags(ctx->client->db, keyname, flags); } else { value = lookupKeyReadWithFlags(ctx->client->db, keyname, flags); if (value == NULL) { return NULL; } } /* Setup the key handle. */ kp = zmalloc(sizeof(*kp)); moduleInitKey(kp, ctx, keyname, value, mode); autoMemoryAdd(ctx, VALKEYMODULE_AM_KEY, kp); return kp; } /** * Returns the full OpenKey modes mask, using the return value * the module can check if a certain set of OpenKey modes are supported * by the server version in use. * Example: * * int supportedMode = VM_GetOpenKeyModesAll(); * if (supportedMode & VALKEYMODULE_OPEN_KEY_NOTOUCH) { * // VALKEYMODULE_OPEN_KEY_NOTOUCH is supported * } else{ * // VALKEYMODULE_OPEN_KEY_NOTOUCH is not supported * } */ int VM_GetOpenKeyModesAll(void) { return _VALKEYMODULE_OPEN_KEY_ALL; } /* Destroy a ValkeyModuleKey struct (freeing is the responsibility of the caller). */ static void moduleCloseKey(ValkeyModuleKey *key) { int signal = SHOULD_SIGNAL_MODIFIED_KEYS(key->ctx); if ((key->mode & VALKEYMODULE_WRITE) && signal) signalModifiedKey(key->ctx->client, key->db, key->key); if (key->value) { if (key->iter) moduleFreeKeyIterator(key); switch (key->value->type) { case OBJ_ZSET: VM_ZsetRangeStop(key); break; case OBJ_STREAM: if (key->u.stream.signalready) /* One or more VM_StreamAdd() have been done. */ signalKeyAsReady(key->db, key->key, OBJ_STREAM); break; } } serverAssert(key->iter == NULL); decrRefCount(key->key); } /* Close a key handle. */ void VM_CloseKey(ValkeyModuleKey *key) { if (key == NULL) return; moduleCloseKey(key); autoMemoryFreed(key->ctx, VALKEYMODULE_AM_KEY, key); zfree(key); } /* Return the type of the key. If the key pointer is NULL then * VALKEYMODULE_KEYTYPE_EMPTY is returned. */ int VM_KeyType(ValkeyModuleKey *key) { if (key == NULL || key->value == NULL) return VALKEYMODULE_KEYTYPE_EMPTY; /* We map between defines so that we are free to change the internal * defines as desired. */ switch (key->value->type) { case OBJ_STRING: return VALKEYMODULE_KEYTYPE_STRING; case OBJ_LIST: return VALKEYMODULE_KEYTYPE_LIST; case OBJ_SET: return VALKEYMODULE_KEYTYPE_SET; case OBJ_ZSET: return VALKEYMODULE_KEYTYPE_ZSET; case OBJ_HASH: return VALKEYMODULE_KEYTYPE_HASH; case OBJ_MODULE: return VALKEYMODULE_KEYTYPE_MODULE; case OBJ_STREAM: return VALKEYMODULE_KEYTYPE_STREAM; default: return VALKEYMODULE_KEYTYPE_EMPTY; } } /* Return the length of the value associated with the key. * For strings this is the length of the string. For all the other types * is the number of elements (just counting keys for hashes). * * If the key pointer is NULL or the key is empty, zero is returned. */ size_t VM_ValueLength(ValkeyModuleKey *key) { if (key == NULL || key->value == NULL) return 0; switch (key->value->type) { case OBJ_STRING: return stringObjectLen(key->value); case OBJ_LIST: return listTypeLength(key->value); case OBJ_SET: return setTypeSize(key->value); case OBJ_ZSET: return zsetLength(key->value); case OBJ_HASH: return hashTypeLength(key->value); case OBJ_STREAM: return streamLength(key->value); default: return 0; } } /* If the key is open for writing, remove it, and setup the key to * accept new writes as an empty key (that will be created on demand). * On success VALKEYMODULE_OK is returned. If the key is not open for * writing VALKEYMODULE_ERR is returned. */ int VM_DeleteKey(ValkeyModuleKey *key) { if (!(key->mode & VALKEYMODULE_WRITE)) return VALKEYMODULE_ERR; if (key->value) { dbDelete(key->db, key->key); key->value = NULL; } return VALKEYMODULE_OK; } /* If the key is open for writing, unlink it (that is delete it in a * non-blocking way, not reclaiming memory immediately) and setup the key to * accept new writes as an empty key (that will be created on demand). * On success VALKEYMODULE_OK is returned. If the key is not open for * writing VALKEYMODULE_ERR is returned. */ int VM_UnlinkKey(ValkeyModuleKey *key) { if (!(key->mode & VALKEYMODULE_WRITE)) return VALKEYMODULE_ERR; if (key->value) { dbAsyncDelete(key->db, key->key); key->value = NULL; } return VALKEYMODULE_OK; } /* Return the key expire value, as milliseconds of remaining TTL. * If no TTL is associated with the key or if the key is empty, * VALKEYMODULE_NO_EXPIRE is returned. */ mstime_t VM_GetExpire(ValkeyModuleKey *key) { mstime_t expire = getExpire(key->db, key->key); if (expire == -1 || key->value == NULL) return VALKEYMODULE_NO_EXPIRE; expire -= commandTimeSnapshot(); return expire >= 0 ? expire : 0; } /* Set a new expire for the key. If the special expire * VALKEYMODULE_NO_EXPIRE is set, the expire is cancelled if there was * one (the same as the PERSIST command). * * Note that the expire must be provided as a positive integer representing * the number of milliseconds of TTL the key should have. * * The function returns VALKEYMODULE_OK on success or VALKEYMODULE_ERR if * the key was not open for writing or is an empty key. */ int VM_SetExpire(ValkeyModuleKey *key, mstime_t expire) { if (!(key->mode & VALKEYMODULE_WRITE) || key->value == NULL || (expire < 0 && expire != VALKEYMODULE_NO_EXPIRE)) return VALKEYMODULE_ERR; if (expire != VALKEYMODULE_NO_EXPIRE) { expire += commandTimeSnapshot(); key->value = setExpire(key->ctx->client, key->db, key->key, expire); } else { removeExpire(key->db, key->key); } return VALKEYMODULE_OK; } /* Return the key expire value, as absolute Unix timestamp. * If no TTL is associated with the key or if the key is empty, * VALKEYMODULE_NO_EXPIRE is returned. */ mstime_t VM_GetAbsExpire(ValkeyModuleKey *key) { mstime_t expire = getExpire(key->db, key->key); if (expire == -1 || key->value == NULL) return VALKEYMODULE_NO_EXPIRE; return expire; } /* Set a new expire for the key. If the special expire * VALKEYMODULE_NO_EXPIRE is set, the expire is cancelled if there was * one (the same as the PERSIST command). * * Note that the expire must be provided as a positive integer representing * the absolute Unix timestamp the key should have. * * The function returns VALKEYMODULE_OK on success or VALKEYMODULE_ERR if * the key was not open for writing or is an empty key. */ int VM_SetAbsExpire(ValkeyModuleKey *key, mstime_t expire) { if (!(key->mode & VALKEYMODULE_WRITE) || key->value == NULL || (expire < 0 && expire != VALKEYMODULE_NO_EXPIRE)) return VALKEYMODULE_ERR; if (expire != VALKEYMODULE_NO_EXPIRE) { key->value = setExpire(key->ctx->client, key->db, key->key, expire); } else { removeExpire(key->db, key->key); } return VALKEYMODULE_OK; } /* Performs similar operation to FLUSHALL, and optionally start a new AOF file (if enabled) * If restart_aof is true, you must make sure the command that triggered this call is not * propagated to the AOF file. * When async is set to true, db contents will be freed by a background thread. */ void VM_ResetDataset(int restart_aof, int async) { if (restart_aof && server.aof_state != AOF_OFF) stopAppendOnly(); flushAllDataAndResetRDB((async ? EMPTYDB_ASYNC : EMPTYDB_NO_FLAGS) | EMPTYDB_NOFUNCTIONS); if (server.aof_enabled && restart_aof) restartAOFAfterSYNC(); } /* Returns the number of keys in the current db. */ unsigned long long VM_DbSize(ValkeyModuleCtx *ctx) { return dbSize(ctx->client->db); } /* Returns a name of a random key, or NULL if current db is empty. */ ValkeyModuleString *VM_RandomKey(ValkeyModuleCtx *ctx) { robj *key = dbRandomKey(ctx->client->db); autoMemoryAdd(ctx, VALKEYMODULE_AM_STRING, key); return key; } /* Returns the name of the key currently being processed. */ const ValkeyModuleString *VM_GetKeyNameFromOptCtx(ValkeyModuleKeyOptCtx *ctx) { return ctx->from_key; } /* Returns the name of the target key currently being processed. */ const ValkeyModuleString *VM_GetToKeyNameFromOptCtx(ValkeyModuleKeyOptCtx *ctx) { return ctx->to_key; } /* Returns the dbid currently being processed. */ int VM_GetDbIdFromOptCtx(ValkeyModuleKeyOptCtx *ctx) { return ctx->from_dbid; } /* Returns the target dbid currently being processed. */ int VM_GetToDbIdFromOptCtx(ValkeyModuleKeyOptCtx *ctx) { return ctx->to_dbid; } /* -------------------------------------------------------------------------- * ## Key API for String type * * See also VM_ValueLength(), which returns the length of a string. * -------------------------------------------------------------------------- */ /* If the key is open for writing, set the specified string 'str' as the * value of the key, deleting the old value if any. * On success VALKEYMODULE_OK is returned. If the key is not open for * writing or there is an active iterator, VALKEYMODULE_ERR is returned. */ int VM_StringSet(ValkeyModuleKey *key, ValkeyModuleString *str) { if (!(key->mode & VALKEYMODULE_WRITE) || key->iter) return VALKEYMODULE_ERR; VM_DeleteKey(key); /* Retain str so setKey copies it to db rather than reallocating it. */ incrRefCount(str); setKey(key->ctx->client, key->db, key->key, &str, SETKEY_NO_SIGNAL); key->value = str; return VALKEYMODULE_OK; } /* Prepare the key associated string value for DMA access, and returns * a pointer and size (by reference), that the user can use to read or * modify the string in-place accessing it directly via pointer. * * The 'mode' is composed by bitwise OR-ing the following flags: * * VALKEYMODULE_READ -- Read access * VALKEYMODULE_WRITE -- Write access * * If the DMA is not requested for writing, the pointer returned should * only be accessed in a read-only fashion. * * On error (wrong type) NULL is returned. * * DMA access rules: * * 1. No other key writing function should be called since the moment * the pointer is obtained, for all the time we want to use DMA access * to read or modify the string. * * 2. Each time VM_StringTruncate() is called, to continue with the DMA * access, VM_StringDMA() should be called again to re-obtain * a new pointer and length. * * 3. If the returned pointer is not NULL, but the length is zero, no * byte can be touched (the string is empty, or the key itself is empty) * so a VM_StringTruncate() call should be used if there is to enlarge * the string, and later call StringDMA() again to get the pointer. */ char *VM_StringDMA(ValkeyModuleKey *key, size_t *len, int mode) { /* We need to return *some* pointer for empty keys, we just return * a string literal pointer, that is the advantage to be mapped into * a read only memory page, so the module will segfault if a write * attempt is performed. */ char *emptystring = ""; if (key->value == NULL) { *len = 0; return emptystring; } if (key->value->type != OBJ_STRING) return NULL; /* For write access, and even for read access if the object is encoded, * we unshare the string (that has the side effect of decoding it). */ if ((mode & VALKEYMODULE_WRITE) || key->value->encoding != OBJ_ENCODING_RAW) key->value = dbUnshareStringValue(key->db, key->key, key->value); *len = sdslen(key->value->ptr); return key->value->ptr; } /* If the key is open for writing and is of string type, resize it, padding * with zero bytes if the new length is greater than the old one. * * After this call, VM_StringDMA() must be called again to continue * DMA access with the new pointer. * * The function returns VALKEYMODULE_OK on success, and VALKEYMODULE_ERR on * error, that is, the key is not open for writing, is not a string * or resizing for more than 512 MB is requested. * * If the key is empty, a string key is created with the new string value * unless the new length value requested is zero. */ int VM_StringTruncate(ValkeyModuleKey *key, size_t newlen) { if (!(key->mode & VALKEYMODULE_WRITE)) return VALKEYMODULE_ERR; if (key->value && key->value->type != OBJ_STRING) return VALKEYMODULE_ERR; if (newlen > 512 * 1024 * 1024) return VALKEYMODULE_ERR; /* Empty key and new len set to 0. Just return VALKEYMODULE_OK without * doing anything. */ if (key->value == NULL && newlen == 0) return VALKEYMODULE_OK; if (key->value == NULL) { /* Empty key: create it with the new size. */ robj *o = createObject(OBJ_STRING, sdsnewlen(NULL, newlen)); setKey(key->ctx->client, key->db, key->key, &o, SETKEY_NO_SIGNAL); key->value = o; } else { /* Unshare and resize. */ key->value = dbUnshareStringValue(key->db, key->key, key->value); size_t curlen = sdslen(key->value->ptr); if (newlen > curlen) { key->value->ptr = sdsgrowzero(key->value->ptr, newlen); } else if (newlen < curlen) { sdssubstr(key->value->ptr, 0, newlen); /* If the string is too wasteful, reallocate it. */ if (sdslen(key->value->ptr) < sdsavail(key->value->ptr)) key->value->ptr = sdsRemoveFreeSpace(key->value->ptr, 0); } } return VALKEYMODULE_OK; } /* -------------------------------------------------------------------------- * ## Key API for List type * * Many of the list functions access elements by index. Since a list is in * essence a doubly-linked list, accessing elements by index is generally an * O(N) operation. However, if elements are accessed sequentially or with * indices close together, the functions are optimized to seek the index from * the previous index, rather than seeking from the ends of the list. * * This enables iteration to be done efficiently using a simple for loop: * * long n = VM_ValueLength(key); * for (long i = 0; i < n; i++) { * ValkeyModuleString *elem = ValkeyModule_ListGet(key, i); * // Do stuff... * } * * Note that after modifying a list using VM_ListPop, VM_ListSet or * VM_ListInsert, the internal iterator is invalidated so the next operation * will require a linear seek. * * Modifying a list in any another way, for example using VM_Call(), while a key * is open will confuse the internal iterator and may cause trouble if the key * is used after such modifications. The key must be reopened in this case. * * See also VM_ValueLength(), which returns the length of a list. * -------------------------------------------------------------------------- */ /* Seeks the key's internal list iterator to the given index. On success, 1 is * returned and key->iter, key->u.list.entry and key->u.list.index are set. On * failure, 0 is returned and errno is set as required by the list API * functions. */ int moduleListIteratorSeek(ValkeyModuleKey *key, long index, int mode) { if (!key) { errno = EINVAL; return 0; } else if (!key->value || key->value->type != OBJ_LIST) { errno = ENOTSUP; return 0; } if (!(key->mode & mode)) { errno = EBADF; return 0; } long length = listTypeLength(key->value); if (index < -length || index >= length) { errno = EDOM; /* Invalid index */ return 0; } if (key->iter == NULL) { /* No existing iterator. Create one. */ key->iter = listTypeInitIterator(key->value, index, LIST_TAIL); serverAssert(key->iter != NULL); serverAssert(listTypeNext(key->iter, &key->u.list.entry)); key->u.list.index = index; return 1; } /* There's an existing iterator. Make sure the requested index has the same * sign as the iterator's index. */ if (index < 0 && key->u.list.index >= 0) index += length; else if (index >= 0 && key->u.list.index < 0) index -= length; if (index == key->u.list.index) return 1; /* We're done. */ /* Seek the iterator to the requested index. */ unsigned char dir = key->u.list.index < index ? LIST_TAIL : LIST_HEAD; listTypeSetIteratorDirection(key->iter, &key->u.list.entry, dir); while (key->u.list.index != index) { serverAssert(listTypeNext(key->iter, &key->u.list.entry)); key->u.list.index += dir == LIST_HEAD ? -1 : 1; } return 1; } /* Push an element into a list, on head or tail depending on 'where' argument * (VALKEYMODULE_LIST_HEAD or VALKEYMODULE_LIST_TAIL). If the key refers to an * empty key opened for writing, the key is created. On success, VALKEYMODULE_OK * is returned. On failure, VALKEYMODULE_ERR is returned and `errno` is set as * follows: * * - EINVAL if key or ele is NULL. * - ENOTSUP if the key is of another type than list. * - EBADF if the key is not opened for writing. * * Note: Before Redis OSS 7.0, `errno` was not set by this function. */ int VM_ListPush(ValkeyModuleKey *key, int where, ValkeyModuleString *ele) { if (!key || !ele) { errno = EINVAL; return VALKEYMODULE_ERR; } else if (key->value != NULL && key->value->type != OBJ_LIST) { errno = ENOTSUP; return VALKEYMODULE_ERR; } if (!(key->mode & VALKEYMODULE_WRITE)) { errno = EBADF; return VALKEYMODULE_ERR; } if (!(key->mode & VALKEYMODULE_WRITE)) return VALKEYMODULE_ERR; if (key->value && key->value->type != OBJ_LIST) return VALKEYMODULE_ERR; if (key->iter) moduleFreeKeyIterator(key); if (key->value == NULL) moduleCreateEmptyKey(key, VALKEYMODULE_KEYTYPE_LIST); listTypeTryConversionAppend(key->value, &ele, 0, 0, moduleFreeListIterator, key); listTypePush(key->value, ele, (where == VALKEYMODULE_LIST_HEAD) ? LIST_HEAD : LIST_TAIL); return VALKEYMODULE_OK; } /* Pop an element from the list, and returns it as a module string object * that the user should be free with VM_FreeString() or by enabling * automatic memory. The `where` argument specifies if the element should be * popped from the beginning or the end of the list (VALKEYMODULE_LIST_HEAD or * VALKEYMODULE_LIST_TAIL). On failure, the command returns NULL and sets * `errno` as follows: * * - EINVAL if key is NULL. * - ENOTSUP if the key is empty or of another type than list. * - EBADF if the key is not opened for writing. * * Note: Before Redis OSS 7.0, `errno` was not set by this function. */ ValkeyModuleString *VM_ListPop(ValkeyModuleKey *key, int where) { if (!key) { errno = EINVAL; return NULL; } else if (key->value == NULL || key->value->type != OBJ_LIST) { errno = ENOTSUP; return NULL; } else if (!(key->mode & VALKEYMODULE_WRITE)) { errno = EBADF; return NULL; } if (key->iter) moduleFreeKeyIterator(key); robj *ele = listTypePop(key->value, (where == VALKEYMODULE_LIST_HEAD) ? LIST_HEAD : LIST_TAIL); robj *decoded = getDecodedObject(ele); decrRefCount(ele); if (!moduleDelKeyIfEmpty(key)) listTypeTryConversion(key->value, LIST_CONV_SHRINKING, moduleFreeListIterator, key); autoMemoryAdd(key->ctx, VALKEYMODULE_AM_STRING, decoded); return decoded; } /* Returns the element at index `index` in the list stored at `key`, like the * LINDEX command. The element should be free'd using VM_FreeString() or using * automatic memory management. * * The index is zero-based, so 0 means the first element, 1 the second element * and so on. Negative indices can be used to designate elements starting at the * tail of the list. Here, -1 means the last element, -2 means the penultimate * and so forth. * * When no value is found at the given key and index, NULL is returned and * `errno` is set as follows: * * - EINVAL if key is NULL. * - ENOTSUP if the key is not a list. * - EBADF if the key is not opened for reading. * - EDOM if the index is not a valid index in the list. */ ValkeyModuleString *VM_ListGet(ValkeyModuleKey *key, long index) { if (moduleListIteratorSeek(key, index, VALKEYMODULE_READ)) { robj *elem = listTypeGet(&key->u.list.entry); robj *decoded = getDecodedObject(elem); decrRefCount(elem); autoMemoryAdd(key->ctx, VALKEYMODULE_AM_STRING, decoded); return decoded; } else { return NULL; } } /* Replaces the element at index `index` in the list stored at `key`. * * The index is zero-based, so 0 means the first element, 1 the second element * and so on. Negative indices can be used to designate elements starting at the * tail of the list. Here, -1 means the last element, -2 means the penultimate * and so forth. * * On success, VALKEYMODULE_OK is returned. On failure, VALKEYMODULE_ERR is * returned and `errno` is set as follows: * * - EINVAL if key or value is NULL. * - ENOTSUP if the key is not a list. * - EBADF if the key is not opened for writing. * - EDOM if the index is not a valid index in the list. */ int VM_ListSet(ValkeyModuleKey *key, long index, ValkeyModuleString *value) { if (!value) { errno = EINVAL; return VALKEYMODULE_ERR; } if (!key->value || key->value->type != OBJ_LIST) { errno = ENOTSUP; return VALKEYMODULE_ERR; } listTypeTryConversionAppend(key->value, &value, 0, 0, moduleFreeListIterator, key); if (moduleListIteratorSeek(key, index, VALKEYMODULE_WRITE)) { listTypeReplace(&key->u.list.entry, value); /* A note in quicklist.c forbids use of iterator after insert, so * probably also after replace. */ moduleFreeKeyIterator(key); return VALKEYMODULE_OK; } else { return VALKEYMODULE_ERR; } } /* Inserts an element at the given index. * * The index is zero-based, so 0 means the first element, 1 the second element * and so on. Negative indices can be used to designate elements starting at the * tail of the list. Here, -1 means the last element, -2 means the penultimate * and so forth. The index is the element's index after inserting it. * * On success, VALKEYMODULE_OK is returned. On failure, VALKEYMODULE_ERR is * returned and `errno` is set as follows: * * - EINVAL if key or value is NULL. * - ENOTSUP if the key of another type than list. * - EBADF if the key is not opened for writing. * - EDOM if the index is not a valid index in the list. */ int VM_ListInsert(ValkeyModuleKey *key, long index, ValkeyModuleString *value) { if (!value) { errno = EINVAL; return VALKEYMODULE_ERR; } else if (key != NULL && key->value == NULL && (index == 0 || index == -1)) { /* Insert in empty key => push. */ return VM_ListPush(key, VALKEYMODULE_LIST_TAIL, value); } else if (key != NULL && key->value != NULL && key->value->type == OBJ_LIST && (index == (long)listTypeLength(key->value) || index == -1)) { /* Insert after the last element => push tail. */ return VM_ListPush(key, VALKEYMODULE_LIST_TAIL, value); } else if (key != NULL && key->value != NULL && key->value->type == OBJ_LIST && (index == 0 || index == -(long)listTypeLength(key->value) - 1)) { /* Insert before the first element => push head. */ return VM_ListPush(key, VALKEYMODULE_LIST_HEAD, value); } listTypeTryConversionAppend(key->value, &value, 0, 0, moduleFreeListIterator, key); if (moduleListIteratorSeek(key, index, VALKEYMODULE_WRITE)) { int where = index < 0 ? LIST_TAIL : LIST_HEAD; listTypeInsert(&key->u.list.entry, value, where); /* A note in quicklist.c forbids use of iterator after insert. */ moduleFreeKeyIterator(key); return VALKEYMODULE_OK; } else { return VALKEYMODULE_ERR; } } /* Removes an element at the given index. The index is 0-based. A negative index * can also be used, counting from the end of the list. * * On success, VALKEYMODULE_OK is returned. On failure, VALKEYMODULE_ERR is * returned and `errno` is set as follows: * * - EINVAL if key or value is NULL. * - ENOTSUP if the key is not a list. * - EBADF if the key is not opened for writing. * - EDOM if the index is not a valid index in the list. */ int VM_ListDelete(ValkeyModuleKey *key, long index) { if (moduleListIteratorSeek(key, index, VALKEYMODULE_WRITE)) { listTypeDelete(key->iter, &key->u.list.entry); if (moduleDelKeyIfEmpty(key)) return VALKEYMODULE_OK; listTypeTryConversion(key->value, LIST_CONV_SHRINKING, moduleFreeListIterator, key); if (!key->iter) return VALKEYMODULE_OK; /* Return ASAP if iterator has been freed */ if (listTypeNext(key->iter, &key->u.list.entry)) { /* After delete entry at position 'index', we need to update * 'key->u.list.index' according to the following cases: * 1) [1, 2, 3] => dir: forward, index: 0 => [2, 3] => index: still 0 * 2) [1, 2, 3] => dir: forward, index: -3 => [2, 3] => index: -2 * 3) [1, 2, 3] => dir: reverse, index: 2 => [1, 2] => index: 1 * 4) [1, 2, 3] => dir: reverse, index: -1 => [1, 2] => index: still -1 */ listTypeIterator *li = key->iter; int reverse = li->direction == LIST_HEAD; if (key->u.list.index < 0) key->u.list.index += reverse ? 0 : 1; else key->u.list.index += reverse ? -1 : 0; } else { /* Reset list iterator if the next entry doesn't exist. */ moduleFreeKeyIterator(key); } return VALKEYMODULE_OK; } else { return VALKEYMODULE_ERR; } } /* -------------------------------------------------------------------------- * ## Key API for Sorted Set type * * See also VM_ValueLength(), which returns the length of a sorted set. * -------------------------------------------------------------------------- */ /* Conversion from/to public flags of the Modules API and our private flags, * so that we have everything decoupled. */ int moduleZsetAddFlagsToCoreFlags(int flags) { int retflags = 0; if (flags & VALKEYMODULE_ZADD_XX) retflags |= ZADD_IN_XX; if (flags & VALKEYMODULE_ZADD_NX) retflags |= ZADD_IN_NX; if (flags & VALKEYMODULE_ZADD_GT) retflags |= ZADD_IN_GT; if (flags & VALKEYMODULE_ZADD_LT) retflags |= ZADD_IN_LT; return retflags; } /* See previous function comment. */ int moduleZsetAddFlagsFromCoreFlags(int flags) { int retflags = 0; if (flags & ZADD_OUT_ADDED) retflags |= VALKEYMODULE_ZADD_ADDED; if (flags & ZADD_OUT_UPDATED) retflags |= VALKEYMODULE_ZADD_UPDATED; if (flags & ZADD_OUT_NOP) retflags |= VALKEYMODULE_ZADD_NOP; return retflags; } /* Add a new element into a sorted set, with the specified 'score'. * If the element already exists, the score is updated. * * A new sorted set is created at value if the key is an empty open key * setup for writing. * * Additional flags can be passed to the function via a pointer, the flags * are both used to receive input and to communicate state when the function * returns. 'flagsptr' can be NULL if no special flags are used. * * The input flags are: * * VALKEYMODULE_ZADD_XX: Element must already exist. Do nothing otherwise. * VALKEYMODULE_ZADD_NX: Element must not exist. Do nothing otherwise. * VALKEYMODULE_ZADD_GT: If element exists, new score must be greater than the current score. * Do nothing otherwise. Can optionally be combined with XX. * VALKEYMODULE_ZADD_LT: If element exists, new score must be less than the current score. * Do nothing otherwise. Can optionally be combined with XX. * * The output flags are: * * VALKEYMODULE_ZADD_ADDED: The new element was added to the sorted set. * VALKEYMODULE_ZADD_UPDATED: The score of the element was updated. * VALKEYMODULE_ZADD_NOP: No operation was performed because XX or NX flags. * * On success the function returns VALKEYMODULE_OK. On the following errors * VALKEYMODULE_ERR is returned: * * * The key was not opened for writing. * * The key is of the wrong type. * * 'score' double value is not a number (NaN). */ int VM_ZsetAdd(ValkeyModuleKey *key, double score, ValkeyModuleString *ele, int *flagsptr) { int in_flags = 0, out_flags = 0; if (!(key->mode & VALKEYMODULE_WRITE)) return VALKEYMODULE_ERR; if (key->value && key->value->type != OBJ_ZSET) return VALKEYMODULE_ERR; if (key->value == NULL) moduleCreateEmptyKey(key, VALKEYMODULE_KEYTYPE_ZSET); if (flagsptr) in_flags = moduleZsetAddFlagsToCoreFlags(*flagsptr); if (zsetAdd(key->value, score, ele->ptr, in_flags, &out_flags, NULL) == 0) { if (flagsptr) *flagsptr = 0; moduleDelKeyIfEmpty(key); return VALKEYMODULE_ERR; } if (flagsptr) *flagsptr = moduleZsetAddFlagsFromCoreFlags(out_flags); return VALKEYMODULE_OK; } /* This function works exactly like VM_ZsetAdd(), but instead of setting * a new score, the score of the existing element is incremented, or if the * element does not already exist, it is added assuming the old score was * zero. * * The input and output flags, and the return value, have the same exact * meaning, with the only difference that this function will return * VALKEYMODULE_ERR even when 'score' is a valid double number, but adding it * to the existing score results into a NaN (not a number) condition. * * This function has an additional field 'newscore', if not NULL is filled * with the new score of the element after the increment, if no error * is returned. */ int VM_ZsetIncrby(ValkeyModuleKey *key, double score, ValkeyModuleString *ele, int *flagsptr, double *newscore) { int in_flags = 0, out_flags = 0; if (!(key->mode & VALKEYMODULE_WRITE)) return VALKEYMODULE_ERR; if (key->value && key->value->type != OBJ_ZSET) return VALKEYMODULE_ERR; if (key->value == NULL) moduleCreateEmptyKey(key, VALKEYMODULE_KEYTYPE_ZSET); if (flagsptr) in_flags = moduleZsetAddFlagsToCoreFlags(*flagsptr); in_flags |= ZADD_IN_INCR; if (zsetAdd(key->value, score, ele->ptr, in_flags, &out_flags, newscore) == 0) { if (flagsptr) *flagsptr = 0; moduleDelKeyIfEmpty(key); return VALKEYMODULE_ERR; } if (flagsptr) *flagsptr = moduleZsetAddFlagsFromCoreFlags(out_flags); return VALKEYMODULE_OK; } /* Remove the specified element from the sorted set. * The function returns VALKEYMODULE_OK on success, and VALKEYMODULE_ERR * on one of the following conditions: * * * The key was not opened for writing. * * The key is of the wrong type. * * The return value does NOT indicate the fact the element was really * removed (since it existed) or not, just if the function was executed * with success. * * In order to know if the element was removed, the additional argument * 'deleted' must be passed, that populates the integer by reference * setting it to 1 or 0 depending on the outcome of the operation. * The 'deleted' argument can be NULL if the caller is not interested * to know if the element was really removed. * * Empty keys will be handled correctly by doing nothing. */ int VM_ZsetRem(ValkeyModuleKey *key, ValkeyModuleString *ele, int *deleted) { if (!(key->mode & VALKEYMODULE_WRITE)) return VALKEYMODULE_ERR; if (key->value && key->value->type != OBJ_ZSET) return VALKEYMODULE_ERR; if (key->value != NULL && zsetDel(key->value, ele->ptr)) { if (deleted) *deleted = 1; moduleDelKeyIfEmpty(key); } else { if (deleted) *deleted = 0; } return VALKEYMODULE_OK; } /* On success retrieve the double score associated at the sorted set element * 'ele' and returns VALKEYMODULE_OK. Otherwise VALKEYMODULE_ERR is returned * to signal one of the following conditions: * * * There is no such element 'ele' in the sorted set. * * The key is not a sorted set. * * The key is an open empty key. */ int VM_ZsetScore(ValkeyModuleKey *key, ValkeyModuleString *ele, double *score) { if (key->value == NULL) return VALKEYMODULE_ERR; if (key->value->type != OBJ_ZSET) return VALKEYMODULE_ERR; if (zsetScore(key->value, ele->ptr, score) == C_ERR) return VALKEYMODULE_ERR; return VALKEYMODULE_OK; } /* -------------------------------------------------------------------------- * ## Key API for Sorted Set iterator * -------------------------------------------------------------------------- */ void zsetKeyReset(ValkeyModuleKey *key) { key->u.zset.type = VALKEYMODULE_ZSET_RANGE_NONE; key->u.zset.current = NULL; key->u.zset.er = 1; } /* Stop a sorted set iteration. */ void VM_ZsetRangeStop(ValkeyModuleKey *key) { if (!key->value || key->value->type != OBJ_ZSET) return; /* Free resources if needed. */ if (key->u.zset.type == VALKEYMODULE_ZSET_RANGE_LEX) zslFreeLexRange(&key->u.zset.lrs); /* Setup sensible values so that misused iteration API calls when an * iterator is not active will result into something more sensible * than crashing. */ zsetKeyReset(key); } /* Return the "End of range" flag value to signal the end of the iteration. */ int VM_ZsetRangeEndReached(ValkeyModuleKey *key) { if (!key->value || key->value->type != OBJ_ZSET) return 1; return key->u.zset.er; } /* Helper function for VM_ZsetFirstInScoreRange() and VM_ZsetLastInScoreRange(). * Setup the sorted set iteration according to the specified score range * (see the functions calling it for more info). If 'first' is true the * first element in the range is used as a starting point for the iterator * otherwise the last. Return VALKEYMODULE_OK on success otherwise * VALKEYMODULE_ERR. */ int zsetInitScoreRange(ValkeyModuleKey *key, double min, double max, int minex, int maxex, int first) { if (!key->value || key->value->type != OBJ_ZSET) return VALKEYMODULE_ERR; VM_ZsetRangeStop(key); key->u.zset.type = VALKEYMODULE_ZSET_RANGE_SCORE; key->u.zset.er = 0; /* Setup the range structure used by the sorted set core implementation * in order to seek at the specified element. */ zrangespec *zrs = &key->u.zset.rs; zrs->min = min; zrs->max = max; zrs->minex = minex; zrs->maxex = maxex; if (key->value->encoding == OBJ_ENCODING_LISTPACK) { key->u.zset.current = first ? zzlFirstInRange(key->value->ptr, zrs) : zzlLastInRange(key->value->ptr, zrs); } else if (key->value->encoding == OBJ_ENCODING_SKIPLIST) { zset *zs = key->value->ptr; zskiplist *zsl = zs->zsl; key->u.zset.current = first ? zslNthInRange(zsl, zrs, 0) : zslNthInRange(zsl, zrs, -1); } else { serverPanic("Unsupported zset encoding"); } if (key->u.zset.current == NULL) key->u.zset.er = 1; return VALKEYMODULE_OK; } /* Setup a sorted set iterator seeking the first element in the specified * range. Returns VALKEYMODULE_OK if the iterator was correctly initialized * otherwise VALKEYMODULE_ERR is returned in the following conditions: * * 1. The value stored at key is not a sorted set or the key is empty. * * The range is specified according to the two double values 'min' and 'max'. * Both can be infinite using the following two macros: * * * VALKEYMODULE_POSITIVE_INFINITE for positive infinite value * * VALKEYMODULE_NEGATIVE_INFINITE for negative infinite value * * 'minex' and 'maxex' parameters, if true, respectively setup a range * where the min and max value are exclusive (not included) instead of * inclusive. */ int VM_ZsetFirstInScoreRange(ValkeyModuleKey *key, double min, double max, int minex, int maxex) { return zsetInitScoreRange(key, min, max, minex, maxex, 1); } /* Exactly like ValkeyModule_ZsetFirstInScoreRange() but the last element of * the range is selected for the start of the iteration instead. */ int VM_ZsetLastInScoreRange(ValkeyModuleKey *key, double min, double max, int minex, int maxex) { return zsetInitScoreRange(key, min, max, minex, maxex, 0); } /* Helper function for VM_ZsetFirstInLexRange() and VM_ZsetLastInLexRange(). * Setup the sorted set iteration according to the specified lexicographical * range (see the functions calling it for more info). If 'first' is true the * first element in the range is used as a starting point for the iterator * otherwise the last. Return VALKEYMODULE_OK on success otherwise * VALKEYMODULE_ERR. * * Note that this function takes 'min' and 'max' in the same form of the * ZRANGEBYLEX command. */ int zsetInitLexRange(ValkeyModuleKey *key, ValkeyModuleString *min, ValkeyModuleString *max, int first) { if (!key->value || key->value->type != OBJ_ZSET) return VALKEYMODULE_ERR; VM_ZsetRangeStop(key); key->u.zset.er = 0; /* Setup the range structure used by the sorted set core implementation * in order to seek at the specified element. */ zlexrangespec *zlrs = &key->u.zset.lrs; if (zslParseLexRange(min, max, zlrs) == C_ERR) return VALKEYMODULE_ERR; /* Set the range type to lex only after successfully parsing the range, * otherwise we don't want the zlexrangespec to be freed. */ key->u.zset.type = VALKEYMODULE_ZSET_RANGE_LEX; if (key->value->encoding == OBJ_ENCODING_LISTPACK) { key->u.zset.current = first ? zzlFirstInLexRange(key->value->ptr, zlrs) : zzlLastInLexRange(key->value->ptr, zlrs); } else if (key->value->encoding == OBJ_ENCODING_SKIPLIST) { zset *zs = key->value->ptr; zskiplist *zsl = zs->zsl; key->u.zset.current = first ? zslNthInLexRange(zsl, zlrs, 0) : zslNthInLexRange(zsl, zlrs, -1); } else { serverPanic("Unsupported zset encoding"); } if (key->u.zset.current == NULL) key->u.zset.er = 1; return VALKEYMODULE_OK; } /* Setup a sorted set iterator seeking the first element in the specified * lexicographical range. Returns VALKEYMODULE_OK if the iterator was correctly * initialized otherwise VALKEYMODULE_ERR is returned in the * following conditions: * * 1. The value stored at key is not a sorted set or the key is empty. * 2. The lexicographical range 'min' and 'max' format is invalid. * * 'min' and 'max' should be provided as two ValkeyModuleString objects * in the same format as the parameters passed to the ZRANGEBYLEX command. * The function does not take ownership of the objects, so they can be released * ASAP after the iterator is setup. */ int VM_ZsetFirstInLexRange(ValkeyModuleKey *key, ValkeyModuleString *min, ValkeyModuleString *max) { return zsetInitLexRange(key, min, max, 1); } /* Exactly like ValkeyModule_ZsetFirstInLexRange() but the last element of * the range is selected for the start of the iteration instead. */ int VM_ZsetLastInLexRange(ValkeyModuleKey *key, ValkeyModuleString *min, ValkeyModuleString *max) { return zsetInitLexRange(key, min, max, 0); } /* Return the current sorted set element of an active sorted set iterator * or NULL if the range specified in the iterator does not include any * element. */ ValkeyModuleString *VM_ZsetRangeCurrentElement(ValkeyModuleKey *key, double *score) { ValkeyModuleString *str; if (!key->value || key->value->type != OBJ_ZSET) return NULL; if (key->u.zset.current == NULL) return NULL; if (key->value->encoding == OBJ_ENCODING_LISTPACK) { unsigned char *eptr, *sptr; eptr = key->u.zset.current; sds ele = lpGetObject(eptr); if (score) { sptr = lpNext(key->value->ptr, eptr); *score = zzlGetScore(sptr); } str = createObject(OBJ_STRING, ele); } else if (key->value->encoding == OBJ_ENCODING_SKIPLIST) { zskiplistNode *ln = key->u.zset.current; if (score) *score = ln->score; str = createStringObject(ln->ele, sdslen(ln->ele)); } else { serverPanic("Unsupported zset encoding"); } autoMemoryAdd(key->ctx, VALKEYMODULE_AM_STRING, str); return str; } /* Go to the next element of the sorted set iterator. Returns 1 if there was * a next element, 0 if we are already at the latest element or the range * does not include any item at all. */ int VM_ZsetRangeNext(ValkeyModuleKey *key) { if (!key->value || key->value->type != OBJ_ZSET) return 0; if (!key->u.zset.type || !key->u.zset.current) return 0; /* No active iterator. */ if (key->value->encoding == OBJ_ENCODING_LISTPACK) { unsigned char *zl = key->value->ptr; unsigned char *eptr = key->u.zset.current; unsigned char *next; next = lpNext(zl, eptr); /* Skip element. */ if (next) next = lpNext(zl, next); /* Skip score. */ if (next == NULL) { key->u.zset.er = 1; return 0; } else { /* Are we still within the range? */ if (key->u.zset.type == VALKEYMODULE_ZSET_RANGE_SCORE) { /* Fetch the next element score for the * range check. */ unsigned char *saved_next = next; next = lpNext(zl, next); /* Skip next element. */ double score = zzlGetScore(next); /* Obtain the next score. */ if (!zslValueLteMax(score, &key->u.zset.rs)) { key->u.zset.er = 1; return 0; } next = saved_next; } else if (key->u.zset.type == VALKEYMODULE_ZSET_RANGE_LEX) { if (!zzlLexValueLteMax(next, &key->u.zset.lrs)) { key->u.zset.er = 1; return 0; } } key->u.zset.current = next; return 1; } } else if (key->value->encoding == OBJ_ENCODING_SKIPLIST) { zskiplistNode *ln = key->u.zset.current, *next = ln->level[0].forward; if (next == NULL) { key->u.zset.er = 1; return 0; } else { /* Are we still within the range? */ if (key->u.zset.type == VALKEYMODULE_ZSET_RANGE_SCORE && !zslValueLteMax(next->score, &key->u.zset.rs)) { key->u.zset.er = 1; return 0; } else if (key->u.zset.type == VALKEYMODULE_ZSET_RANGE_LEX) { if (!zslLexValueLteMax(next->ele, &key->u.zset.lrs)) { key->u.zset.er = 1; return 0; } } key->u.zset.current = next; return 1; } } else { serverPanic("Unsupported zset encoding"); } } /* Go to the previous element of the sorted set iterator. Returns 1 if there was * a previous element, 0 if we are already at the first element or the range * does not include any item at all. */ int VM_ZsetRangePrev(ValkeyModuleKey *key) { if (!key->value || key->value->type != OBJ_ZSET) return 0; if (!key->u.zset.type || !key->u.zset.current) return 0; /* No active iterator. */ if (key->value->encoding == OBJ_ENCODING_LISTPACK) { unsigned char *zl = key->value->ptr; unsigned char *eptr = key->u.zset.current; unsigned char *prev; prev = lpPrev(zl, eptr); /* Go back to previous score. */ if (prev) prev = lpPrev(zl, prev); /* Back to previous ele. */ if (prev == NULL) { key->u.zset.er = 1; return 0; } else { /* Are we still within the range? */ if (key->u.zset.type == VALKEYMODULE_ZSET_RANGE_SCORE) { /* Fetch the previous element score for the * range check. */ unsigned char *saved_prev = prev; prev = lpNext(zl, prev); /* Skip element to get the score.*/ double score = zzlGetScore(prev); /* Obtain the prev score. */ if (!zslValueGteMin(score, &key->u.zset.rs)) { key->u.zset.er = 1; return 0; } prev = saved_prev; } else if (key->u.zset.type == VALKEYMODULE_ZSET_RANGE_LEX) { if (!zzlLexValueGteMin(prev, &key->u.zset.lrs)) { key->u.zset.er = 1; return 0; } } key->u.zset.current = prev; return 1; } } else if (key->value->encoding == OBJ_ENCODING_SKIPLIST) { zskiplistNode *ln = key->u.zset.current, *prev = ln->backward; if (prev == NULL) { key->u.zset.er = 1; return 0; } else { /* Are we still within the range? */ if (key->u.zset.type == VALKEYMODULE_ZSET_RANGE_SCORE && !zslValueGteMin(prev->score, &key->u.zset.rs)) { key->u.zset.er = 1; return 0; } else if (key->u.zset.type == VALKEYMODULE_ZSET_RANGE_LEX) { if (!zslLexValueGteMin(prev->ele, &key->u.zset.lrs)) { key->u.zset.er = 1; return 0; } } key->u.zset.current = prev; return 1; } } else { serverPanic("Unsupported zset encoding"); } } /* -------------------------------------------------------------------------- * ## Key API for Hash type * * See also VM_ValueLength(), which returns the number of fields in a hash. * -------------------------------------------------------------------------- */ /* Set the field of the specified hash field to the specified value. * If the key is an empty key open for writing, it is created with an empty * hash value, in order to set the specified field. * * The function is variadic and the user must specify pairs of field * names and values, both as ValkeyModuleString pointers (unless the * CFIELD option is set, see later). At the end of the field/value-ptr pairs, * NULL must be specified as last argument to signal the end of the arguments * in the variadic function. * * Example to set the hash argv[1] to the value argv[2]: * * ValkeyModule_HashSet(key,VALKEYMODULE_HASH_NONE,argv[1],argv[2],NULL); * * The function can also be used in order to delete fields (if they exist) * by setting them to the specified value of VALKEYMODULE_HASH_DELETE: * * ValkeyModule_HashSet(key,VALKEYMODULE_HASH_NONE,argv[1], * VALKEYMODULE_HASH_DELETE,NULL); * * The behavior of the command changes with the specified flags, that can be * set to VALKEYMODULE_HASH_NONE if no special behavior is needed. * * VALKEYMODULE_HASH_NX: The operation is performed only if the field was not * already existing in the hash. * VALKEYMODULE_HASH_XX: The operation is performed only if the field was * already existing, so that a new value could be * associated to an existing filed, but no new fields * are created. * VALKEYMODULE_HASH_CFIELDS: The field names passed are null terminated C * strings instead of ValkeyModuleString objects. * VALKEYMODULE_HASH_COUNT_ALL: Include the number of inserted fields in the * returned number, in addition to the number of * updated and deleted fields. (Added in Redis OSS * 6.2.) * * Unless NX is specified, the command overwrites the old field value with * the new one. * * When using VALKEYMODULE_HASH_CFIELDS, field names are reported using * normal C strings, so for example to delete the field "foo" the following * code can be used: * * ValkeyModule_HashSet(key,VALKEYMODULE_HASH_CFIELDS,"foo", * VALKEYMODULE_HASH_DELETE,NULL); * * Return value: * * The number of fields existing in the hash prior to the call, which have been * updated (its old value has been replaced by a new value) or deleted. If the * flag VALKEYMODULE_HASH_COUNT_ALL is set, inserted fields not previously * existing in the hash are also counted. * * If the return value is zero, `errno` is set (since Redis OSS 6.2) as follows: * * - EINVAL if any unknown flags are set or if key is NULL. * - ENOTSUP if the key is associated with a non Hash value. * - EBADF if the key was not opened for writing. * - ENOENT if no fields were counted as described under Return value above. * This is not actually an error. The return value can be zero if all fields * were just created and the COUNT_ALL flag was unset, or if changes were held * back due to the NX and XX flags. * * NOTICE: The return value semantics of this function are very different * between Redis OSS 6.2 and older versions. Modules that use it should determine * the server version and handle it accordingly. */ int VM_HashSet(ValkeyModuleKey *key, int flags, ...) { va_list ap; if (!key || (flags & ~(VALKEYMODULE_HASH_NX | VALKEYMODULE_HASH_XX | VALKEYMODULE_HASH_CFIELDS | VALKEYMODULE_HASH_COUNT_ALL))) { errno = EINVAL; return 0; } else if (key->value && key->value->type != OBJ_HASH) { errno = ENOTSUP; return 0; } else if (!(key->mode & VALKEYMODULE_WRITE)) { errno = EBADF; return 0; } if (key->value == NULL) moduleCreateEmptyKey(key, VALKEYMODULE_KEYTYPE_HASH); int count = 0; va_start(ap, flags); while (1) { ValkeyModuleString *field, *value; /* Get the field and value objects. */ if (flags & VALKEYMODULE_HASH_CFIELDS) { char *cfield = va_arg(ap, char *); if (cfield == NULL) break; field = createRawStringObject(cfield, strlen(cfield)); } else { field = va_arg(ap, ValkeyModuleString *); if (field == NULL) break; } value = va_arg(ap, ValkeyModuleString *); /* Handle XX and NX */ if (flags & (VALKEYMODULE_HASH_XX | VALKEYMODULE_HASH_NX)) { int exists = hashTypeExists(key->value, field->ptr); if (((flags & VALKEYMODULE_HASH_XX) && !exists) || ((flags & VALKEYMODULE_HASH_NX) && exists)) { if (flags & VALKEYMODULE_HASH_CFIELDS) decrRefCount(field); continue; } } /* Handle deletion if value is VALKEYMODULE_HASH_DELETE. */ if (value == VALKEYMODULE_HASH_DELETE) { count += hashTypeDelete(key->value, field->ptr); if (flags & VALKEYMODULE_HASH_CFIELDS) decrRefCount(field); continue; } int low_flags = HASH_SET_COPY; /* If CFIELDS is active, we can pass the ownership of the * SDS object to the low level function that sets the field * to avoid a useless copy. */ if (flags & VALKEYMODULE_HASH_CFIELDS) low_flags |= HASH_SET_TAKE_FIELD; robj *argv[2] = {field, value}; hashTypeTryConversion(key->value, argv, 0, 1); int updated = hashTypeSet(key->value, field->ptr, value->ptr, low_flags); count += (flags & VALKEYMODULE_HASH_COUNT_ALL) ? 1 : updated; /* If CFIELDS is active, SDS string ownership is now of hashTypeSet(), * however we still have to release the 'field' object shell. */ if (flags & VALKEYMODULE_HASH_CFIELDS) { field->ptr = NULL; /* Prevent the SDS string from being freed. */ decrRefCount(field); } } va_end(ap); moduleDelKeyIfEmpty(key); if (count == 0) errno = ENOENT; return count; } /* Get fields from a hash value. This function is called using a variable * number of arguments, alternating a field name (as a ValkeyModuleString * pointer) with a pointer to a ValkeyModuleString pointer, that is set to the * value of the field if the field exists, or NULL if the field does not exist. * At the end of the field/value-ptr pairs, NULL must be specified as last * argument to signal the end of the arguments in the variadic function. * * This is an example usage: * * ValkeyModuleString *first, *second; * ValkeyModule_HashGet(mykey,VALKEYMODULE_HASH_NONE,argv[1],&first, * argv[2],&second,NULL); * * As with ValkeyModule_HashSet() the behavior of the command can be specified * passing flags different than VALKEYMODULE_HASH_NONE: * * VALKEYMODULE_HASH_CFIELDS: field names as null terminated C strings. * * VALKEYMODULE_HASH_EXISTS: instead of setting the value of the field * expecting a ValkeyModuleString pointer to pointer, the function just * reports if the field exists or not and expects an integer pointer * as the second element of each pair. * * Example of VALKEYMODULE_HASH_CFIELDS: * * ValkeyModuleString *username, *hashedpass; * ValkeyModule_HashGet(mykey,VALKEYMODULE_HASH_CFIELDS,"username",&username,"hp",&hashedpass, NULL); * * Example of VALKEYMODULE_HASH_EXISTS: * * int exists; * ValkeyModule_HashGet(mykey,VALKEYMODULE_HASH_EXISTS,argv[1],&exists,NULL); * * The function returns VALKEYMODULE_OK on success and VALKEYMODULE_ERR if * the key is not a hash value. * * Memory management: * * The returned ValkeyModuleString objects should be released with * ValkeyModule_FreeString(), or by enabling automatic memory management. */ int VM_HashGet(ValkeyModuleKey *key, int flags, ...) { va_list ap; if (key->value && key->value->type != OBJ_HASH) return VALKEYMODULE_ERR; va_start(ap, flags); while (1) { ValkeyModuleString *field, **valueptr; int *existsptr; /* Get the field object and the value pointer to pointer. */ if (flags & VALKEYMODULE_HASH_CFIELDS) { char *cfield = va_arg(ap, char *); if (cfield == NULL) break; field = createRawStringObject(cfield, strlen(cfield)); } else { field = va_arg(ap, ValkeyModuleString *); if (field == NULL) break; } /* Query the hash for existence or value object. */ if (flags & VALKEYMODULE_HASH_EXISTS) { existsptr = va_arg(ap, int *); if (key->value) *existsptr = hashTypeExists(key->value, field->ptr); else *existsptr = 0; } else { valueptr = va_arg(ap, ValkeyModuleString **); if (key->value) { *valueptr = hashTypeGetValueObject(key->value, field->ptr); if (*valueptr) { robj *decoded = getDecodedObject(*valueptr); decrRefCount(*valueptr); *valueptr = decoded; } if (*valueptr) autoMemoryAdd(key->ctx, VALKEYMODULE_AM_STRING, *valueptr); } else { *valueptr = NULL; } } /* Cleanup */ if (flags & VALKEYMODULE_HASH_CFIELDS) decrRefCount(field); } va_end(ap); return VALKEYMODULE_OK; } /* -------------------------------------------------------------------------- * ## Key API for Stream type * * For an introduction to streams, see https://valkey.io/topics/streams-intro. * * The type ValkeyModuleStreamID, which is used in stream functions, is a struct * with two 64-bit fields and is defined as * * typedef struct ValkeyModuleStreamID { * uint64_t ms; * uint64_t seq; * } ValkeyModuleStreamID; * * See also VM_ValueLength(), which returns the length of a stream, and the * conversion functions VM_StringToStreamID() and VM_CreateStringFromStreamID(). * -------------------------------------------------------------------------- */ /* Adds an entry to a stream. Like XADD without trimming. * * - `key`: The key where the stream is (or will be) stored * - `flags`: A bit field of * - `VALKEYMODULE_STREAM_ADD_AUTOID`: Assign a stream ID automatically, like * `*` in the XADD command. * - `id`: If the `AUTOID` flag is set, this is where the assigned ID is * returned. Can be NULL if `AUTOID` is set, if you don't care to receive the * ID. If `AUTOID` is not set, this is the requested ID. * - `argv`: A pointer to an array of size `numfields * 2` containing the * fields and values. * - `numfields`: The number of field-value pairs in `argv`. * * Returns VALKEYMODULE_OK if an entry has been added. On failure, * VALKEYMODULE_ERR is returned and `errno` is set as follows: * * - EINVAL if called with invalid arguments * - ENOTSUP if the key refers to a value of a type other than stream * - EBADF if the key was not opened for writing * - EDOM if the given ID was 0-0 or not greater than all other IDs in the * stream (only if the AUTOID flag is unset) * - EFBIG if the stream has reached the last possible ID * - ERANGE if the elements are too large to be stored. */ int VM_StreamAdd(ValkeyModuleKey *key, int flags, ValkeyModuleStreamID *id, ValkeyModuleString **argv, long numfields) { /* Validate args */ if (!key || (numfields != 0 && !argv) || /* invalid key or argv */ (flags & ~(VALKEYMODULE_STREAM_ADD_AUTOID)) || /* invalid flags */ (!(flags & VALKEYMODULE_STREAM_ADD_AUTOID) && !id)) { /* id required */ errno = EINVAL; return VALKEYMODULE_ERR; } else if (key->value && key->value->type != OBJ_STREAM) { errno = ENOTSUP; /* wrong type */ return VALKEYMODULE_ERR; } else if (!(key->mode & VALKEYMODULE_WRITE)) { errno = EBADF; /* key not open for writing */ return VALKEYMODULE_ERR; } else if (!(flags & VALKEYMODULE_STREAM_ADD_AUTOID) && id->ms == 0 && id->seq == 0) { errno = EDOM; /* ID out of range */ return VALKEYMODULE_ERR; } /* Create key if necessary */ int created = 0; if (key->value == NULL) { moduleCreateEmptyKey(key, VALKEYMODULE_KEYTYPE_STREAM); created = 1; } stream *s = key->value->ptr; if (s->last_id.ms == UINT64_MAX && s->last_id.seq == UINT64_MAX) { /* The stream has reached the last possible ID */ errno = EFBIG; return VALKEYMODULE_ERR; } streamID added_id; streamID use_id; streamID *use_id_ptr = NULL; if (!(flags & VALKEYMODULE_STREAM_ADD_AUTOID)) { use_id.ms = id->ms; use_id.seq = id->seq; use_id_ptr = &use_id; } if (streamAppendItem(s, argv, numfields, &added_id, use_id_ptr, 1) == C_ERR) { /* Either the ID not greater than all existing IDs in the stream, or * the elements are too large to be stored. either way, errno is already * set by streamAppendItem. */ if (created) moduleDelKeyIfEmpty(key); return VALKEYMODULE_ERR; } /* Postponed signalKeyAsReady(). Done implicitly by moduleCreateEmptyKey() * so not needed if the stream has just been created. */ if (!created) key->u.stream.signalready = 1; if (id != NULL) { id->ms = added_id.ms; id->seq = added_id.seq; } return VALKEYMODULE_OK; } /* Deletes an entry from a stream. * * - `key`: A key opened for writing, with no stream iterator started. * - `id`: The stream ID of the entry to delete. * * Returns VALKEYMODULE_OK on success. On failure, VALKEYMODULE_ERR is returned * and `errno` is set as follows: * * - EINVAL if called with invalid arguments * - ENOTSUP if the key refers to a value of a type other than stream or if the * key is empty * - EBADF if the key was not opened for writing or if a stream iterator is * associated with the key * - ENOENT if no entry with the given stream ID exists * * See also VM_StreamIteratorDelete() for deleting the current entry while * iterating using a stream iterator. */ int VM_StreamDelete(ValkeyModuleKey *key, ValkeyModuleStreamID *id) { if (!key || !id) { errno = EINVAL; return VALKEYMODULE_ERR; } else if (!key->value || key->value->type != OBJ_STREAM) { errno = ENOTSUP; /* wrong type */ return VALKEYMODULE_ERR; } else if (!(key->mode & VALKEYMODULE_WRITE) || key->iter != NULL) { errno = EBADF; /* key not opened for writing or iterator started */ return VALKEYMODULE_ERR; } stream *s = key->value->ptr; streamID streamid = {id->ms, id->seq}; if (streamDeleteItem(s, &streamid)) { return VALKEYMODULE_OK; } else { errno = ENOENT; /* no entry with this id */ return VALKEYMODULE_ERR; } } /* Sets up a stream iterator. * * - `key`: The stream key opened for reading using ValkeyModule_OpenKey(). * - `flags`: * - `VALKEYMODULE_STREAM_ITERATOR_EXCLUSIVE`: Don't include `start` and `end` * in the iterated range. * - `VALKEYMODULE_STREAM_ITERATOR_REVERSE`: Iterate in reverse order, starting * from the `end` of the range. * - `start`: The lower bound of the range. Use NULL for the beginning of the * stream. * - `end`: The upper bound of the range. Use NULL for the end of the stream. * * Returns VALKEYMODULE_OK on success. On failure, VALKEYMODULE_ERR is returned * and `errno` is set as follows: * * - EINVAL if called with invalid arguments * - ENOTSUP if the key refers to a value of a type other than stream or if the * key is empty * - EBADF if the key was not opened for writing or if a stream iterator is * already associated with the key * - EDOM if `start` or `end` is outside the valid range * * Returns VALKEYMODULE_OK on success and VALKEYMODULE_ERR if the key doesn't * refer to a stream or if invalid arguments were given. * * The stream IDs are retrieved using ValkeyModule_StreamIteratorNextID() and * for each stream ID, the fields and values are retrieved using * ValkeyModule_StreamIteratorNextField(). The iterator is freed by calling * ValkeyModule_StreamIteratorStop(). * * Example (error handling omitted): * * ValkeyModule_StreamIteratorStart(key, 0, startid_ptr, endid_ptr); * ValkeyModuleStreamID id; * long numfields; * while (ValkeyModule_StreamIteratorNextID(key, &id, &numfields) == * VALKEYMODULE_OK) { * ValkeyModuleString *field, *value; * while (ValkeyModule_StreamIteratorNextField(key, &field, &value) == * VALKEYMODULE_OK) { * // * // ... Do stuff ... * // * ValkeyModule_FreeString(ctx, field); * ValkeyModule_FreeString(ctx, value); * } * } * ValkeyModule_StreamIteratorStop(key); */ int VM_StreamIteratorStart(ValkeyModuleKey *key, int flags, ValkeyModuleStreamID *start, ValkeyModuleStreamID *end) { /* check args */ if (!key || (flags & ~(VALKEYMODULE_STREAM_ITERATOR_EXCLUSIVE | VALKEYMODULE_STREAM_ITERATOR_REVERSE))) { errno = EINVAL; /* key missing or invalid flags */ return VALKEYMODULE_ERR; } else if (!key->value || key->value->type != OBJ_STREAM) { errno = ENOTSUP; return VALKEYMODULE_ERR; /* not a stream */ } else if (key->iter) { errno = EBADF; /* iterator already started */ return VALKEYMODULE_ERR; } /* define range for streamIteratorStart() */ streamID lower, upper; if (start) lower = (streamID){start->ms, start->seq}; if (end) upper = (streamID){end->ms, end->seq}; if (flags & VALKEYMODULE_STREAM_ITERATOR_EXCLUSIVE) { if ((start && streamIncrID(&lower) != C_OK) || (end && streamDecrID(&upper) != C_OK)) { errno = EDOM; /* end is 0-0 or start is MAX-MAX? */ return VALKEYMODULE_ERR; } } /* create iterator */ stream *s = key->value->ptr; int rev = flags & VALKEYMODULE_STREAM_ITERATOR_REVERSE; streamIterator *si = zmalloc(sizeof(*si)); streamIteratorStart(si, s, start ? &lower : NULL, end ? &upper : NULL, rev); key->iter = si; key->u.stream.currentid.ms = 0; /* for VM_StreamIteratorDelete() */ key->u.stream.currentid.seq = 0; key->u.stream.numfieldsleft = 0; /* for VM_StreamIteratorNextField() */ return VALKEYMODULE_OK; } /* Stops a stream iterator created using ValkeyModule_StreamIteratorStart() and * reclaims its memory. * * Returns VALKEYMODULE_OK on success. On failure, VALKEYMODULE_ERR is returned * and `errno` is set as follows: * * - EINVAL if called with a NULL key * - ENOTSUP if the key refers to a value of a type other than stream or if the * key is empty * - EBADF if the key was not opened for writing or if no stream iterator is * associated with the key */ int VM_StreamIteratorStop(ValkeyModuleKey *key) { if (!key) { errno = EINVAL; return VALKEYMODULE_ERR; } else if (!key->value || key->value->type != OBJ_STREAM) { errno = ENOTSUP; return VALKEYMODULE_ERR; } else if (!key->iter) { errno = EBADF; return VALKEYMODULE_ERR; } streamIteratorStop(key->iter); zfree(key->iter); key->iter = NULL; return VALKEYMODULE_OK; } /* Finds the next stream entry and returns its stream ID and the number of * fields. * * - `key`: Key for which a stream iterator has been started using * ValkeyModule_StreamIteratorStart(). * - `id`: The stream ID returned. NULL if you don't care. * - `numfields`: The number of fields in the found stream entry. NULL if you * don't care. * * Returns VALKEYMODULE_OK and sets `*id` and `*numfields` if an entry was found. * On failure, VALKEYMODULE_ERR is returned and `errno` is set as follows: * * - EINVAL if called with a NULL key * - ENOTSUP if the key refers to a value of a type other than stream or if the * key is empty * - EBADF if no stream iterator is associated with the key * - ENOENT if there are no more entries in the range of the iterator * * In practice, if VM_StreamIteratorNextID() is called after a successful call * to VM_StreamIteratorStart() and with the same key, it is safe to assume that * an VALKEYMODULE_ERR return value means that there are no more entries. * * Use ValkeyModule_StreamIteratorNextField() to retrieve the fields and values. * See the example at ValkeyModule_StreamIteratorStart(). */ int VM_StreamIteratorNextID(ValkeyModuleKey *key, ValkeyModuleStreamID *id, long *numfields) { if (!key) { errno = EINVAL; return VALKEYMODULE_ERR; } else if (!key->value || key->value->type != OBJ_STREAM) { errno = ENOTSUP; return VALKEYMODULE_ERR; } else if (!key->iter) { errno = EBADF; return VALKEYMODULE_ERR; } streamIterator *si = key->iter; int64_t *num_ptr = &key->u.stream.numfieldsleft; streamID *streamid_ptr = &key->u.stream.currentid; if (streamIteratorGetID(si, streamid_ptr, num_ptr)) { if (id) { id->ms = streamid_ptr->ms; id->seq = streamid_ptr->seq; } if (numfields) *numfields = *num_ptr; return VALKEYMODULE_OK; } else { /* No entry found. */ key->u.stream.currentid.ms = 0; /* for VM_StreamIteratorDelete() */ key->u.stream.currentid.seq = 0; key->u.stream.numfieldsleft = 0; /* for VM_StreamIteratorNextField() */ errno = ENOENT; return VALKEYMODULE_ERR; } } /* Retrieves the next field of the current stream ID and its corresponding value * in a stream iteration. This function should be called repeatedly after calling * ValkeyModule_StreamIteratorNextID() to fetch each field-value pair. * * - `key`: Key where a stream iterator has been started. * - `field_ptr`: This is where the field is returned. * - `value_ptr`: This is where the value is returned. * * Returns VALKEYMODULE_OK and points `*field_ptr` and `*value_ptr` to freshly * allocated ValkeyModuleString objects. The string objects are freed * automatically when the callback finishes if automatic memory is enabled. On * failure, VALKEYMODULE_ERR is returned and `errno` is set as follows: * * - EINVAL if called with a NULL key * - ENOTSUP if the key refers to a value of a type other than stream or if the * key is empty * - EBADF if no stream iterator is associated with the key * - ENOENT if there are no more fields in the current stream entry * * In practice, if VM_StreamIteratorNextField() is called after a successful * call to VM_StreamIteratorNextID() and with the same key, it is safe to assume * that an VALKEYMODULE_ERR return value means that there are no more fields. * * See the example at ValkeyModule_StreamIteratorStart(). */ int VM_StreamIteratorNextField(ValkeyModuleKey *key, ValkeyModuleString **field_ptr, ValkeyModuleString **value_ptr) { if (!key) { errno = EINVAL; return VALKEYMODULE_ERR; } else if (!key->value || key->value->type != OBJ_STREAM) { errno = ENOTSUP; return VALKEYMODULE_ERR; } else if (!key->iter) { errno = EBADF; return VALKEYMODULE_ERR; } else if (key->u.stream.numfieldsleft <= 0) { errno = ENOENT; return VALKEYMODULE_ERR; } streamIterator *si = key->iter; unsigned char *field, *value; int64_t field_len, value_len; streamIteratorGetField(si, &field, &value, &field_len, &value_len); if (field_ptr) { *field_ptr = createRawStringObject((char *)field, field_len); autoMemoryAdd(key->ctx, VALKEYMODULE_AM_STRING, *field_ptr); } if (value_ptr) { *value_ptr = createRawStringObject((char *)value, value_len); autoMemoryAdd(key->ctx, VALKEYMODULE_AM_STRING, *value_ptr); } key->u.stream.numfieldsleft--; return VALKEYMODULE_OK; } /* Deletes the current stream entry while iterating. * * This function can be called after VM_StreamIteratorNextID() or after any * calls to VM_StreamIteratorNextField(). * * Returns VALKEYMODULE_OK on success. On failure, VALKEYMODULE_ERR is returned * and `errno` is set as follows: * * - EINVAL if key is NULL * - ENOTSUP if the key is empty or is of another type than stream * - EBADF if the key is not opened for writing, if no iterator has been started * - ENOENT if the iterator has no current stream entry */ int VM_StreamIteratorDelete(ValkeyModuleKey *key) { if (!key) { errno = EINVAL; return VALKEYMODULE_ERR; } else if (!key->value || key->value->type != OBJ_STREAM) { errno = ENOTSUP; return VALKEYMODULE_ERR; } else if (!(key->mode & VALKEYMODULE_WRITE) || !key->iter) { errno = EBADF; return VALKEYMODULE_ERR; } else if (key->u.stream.currentid.ms == 0 && key->u.stream.currentid.seq == 0) { errno = ENOENT; return VALKEYMODULE_ERR; } streamIterator *si = key->iter; streamIteratorRemoveEntry(si, &key->u.stream.currentid); key->u.stream.currentid.ms = 0; /* Make sure repeated Delete() fails */ key->u.stream.currentid.seq = 0; key->u.stream.numfieldsleft = 0; /* Make sure NextField() fails */ return VALKEYMODULE_OK; } /* Trim a stream by length, similar to XTRIM with MAXLEN. * * - `key`: Key opened for writing. * - `flags`: A bitfield of * - `VALKEYMODULE_STREAM_TRIM_APPROX`: Trim less if it improves performance, * like XTRIM with `~`. * - `length`: The number of stream entries to keep after trimming. * * Returns the number of entries deleted. On failure, a negative value is * returned and `errno` is set as follows: * * - EINVAL if called with invalid arguments * - ENOTSUP if the key is empty or of a type other than stream * - EBADF if the key is not opened for writing */ long long VM_StreamTrimByLength(ValkeyModuleKey *key, int flags, long long length) { if (!key || (flags & ~(VALKEYMODULE_STREAM_TRIM_APPROX)) || length < 0) { errno = EINVAL; return -1; } else if (!key->value || key->value->type != OBJ_STREAM) { errno = ENOTSUP; return -1; } else if (!(key->mode & VALKEYMODULE_WRITE)) { errno = EBADF; return -1; } int approx = flags & VALKEYMODULE_STREAM_TRIM_APPROX ? 1 : 0; return streamTrimByLength((stream *)key->value->ptr, length, approx); } /* Trim a stream by ID, similar to XTRIM with MINID. * * - `key`: Key opened for writing. * - `flags`: A bitfield of * - `VALKEYMODULE_STREAM_TRIM_APPROX`: Trim less if it improves performance, * like XTRIM with `~`. * - `id`: The smallest stream ID to keep after trimming. * * Returns the number of entries deleted. On failure, a negative value is * returned and `errno` is set as follows: * * - EINVAL if called with invalid arguments * - ENOTSUP if the key is empty or of a type other than stream * - EBADF if the key is not opened for writing */ long long VM_StreamTrimByID(ValkeyModuleKey *key, int flags, ValkeyModuleStreamID *id) { if (!key || (flags & ~(VALKEYMODULE_STREAM_TRIM_APPROX)) || !id) { errno = EINVAL; return -1; } else if (!key->value || key->value->type != OBJ_STREAM) { errno = ENOTSUP; return -1; } else if (!(key->mode & VALKEYMODULE_WRITE)) { errno = EBADF; return -1; } int approx = flags & VALKEYMODULE_STREAM_TRIM_APPROX ? 1 : 0; streamID minid = (streamID){id->ms, id->seq}; return streamTrimByID((stream *)key->value->ptr, minid, approx); } /* -------------------------------------------------------------------------- * ## Calling commands from modules * * VM_Call() sends a command to the server. The remaining functions handle the reply. * -------------------------------------------------------------------------- */ void moduleParseCallReply_Int(ValkeyModuleCallReply *reply); void moduleParseCallReply_BulkString(ValkeyModuleCallReply *reply); void moduleParseCallReply_SimpleString(ValkeyModuleCallReply *reply); void moduleParseCallReply_Array(ValkeyModuleCallReply *reply); /* Free a Call reply and all the nested replies it contains if it's an * array. */ void VM_FreeCallReply(ValkeyModuleCallReply *reply) { /* This is a wrapper for the recursive free reply function. This is needed * in order to have the first level function to return on nested replies, * but only if called by the module API. */ ValkeyModuleCtx *ctx = NULL; if (callReplyType(reply) == VALKEYMODULE_REPLY_PROMISE) { ValkeyModuleAsyncRMCallPromise *promise = callReplyGetPrivateData(reply); ctx = promise->ctx; freeValkeyModuleAsyncRMCallPromise(promise); } else { ctx = callReplyGetPrivateData(reply); } freeCallReply(reply); if (ctx) { autoMemoryFreed(ctx, VALKEYMODULE_AM_REPLY, reply); } } /* Return the reply type as one of the following: * * - VALKEYMODULE_REPLY_UNKNOWN * - VALKEYMODULE_REPLY_STRING * - VALKEYMODULE_REPLY_ERROR * - VALKEYMODULE_REPLY_INTEGER * - VALKEYMODULE_REPLY_ARRAY * - VALKEYMODULE_REPLY_NULL * - VALKEYMODULE_REPLY_MAP * - VALKEYMODULE_REPLY_SET * - VALKEYMODULE_REPLY_BOOL * - VALKEYMODULE_REPLY_DOUBLE * - VALKEYMODULE_REPLY_BIG_NUMBER * - VALKEYMODULE_REPLY_VERBATIM_STRING * - VALKEYMODULE_REPLY_ATTRIBUTE * - VALKEYMODULE_REPLY_PROMISE */ int VM_CallReplyType(ValkeyModuleCallReply *reply) { return callReplyType(reply); } /* Return the reply type length, where applicable. */ size_t VM_CallReplyLength(ValkeyModuleCallReply *reply) { return callReplyGetLen(reply); } /* Return the 'idx'-th nested call reply element of an array reply, or NULL * if the reply type is wrong or the index is out of range. */ ValkeyModuleCallReply *VM_CallReplyArrayElement(ValkeyModuleCallReply *reply, size_t idx) { return callReplyGetArrayElement(reply, idx); } /* Return the `long long` of an integer reply. */ long long VM_CallReplyInteger(ValkeyModuleCallReply *reply) { return callReplyGetLongLong(reply); } /* Return the double value of a double reply. */ double VM_CallReplyDouble(ValkeyModuleCallReply *reply) { return callReplyGetDouble(reply); } /* Return the big number value of a big number reply. */ const char *VM_CallReplyBigNumber(ValkeyModuleCallReply *reply, size_t *len) { return callReplyGetBigNumber(reply, len); } /* Return the value of a verbatim string reply, * An optional output argument can be given to get verbatim reply format. */ const char *VM_CallReplyVerbatim(ValkeyModuleCallReply *reply, size_t *len, const char **format) { return callReplyGetVerbatim(reply, len, format); } /* Return the Boolean value of a Boolean reply. */ int VM_CallReplyBool(ValkeyModuleCallReply *reply) { return callReplyGetBool(reply); } /* Return the 'idx'-th nested call reply element of a set reply, or NULL * if the reply type is wrong or the index is out of range. */ ValkeyModuleCallReply *VM_CallReplySetElement(ValkeyModuleCallReply *reply, size_t idx) { return callReplyGetSetElement(reply, idx); } /* Retrieve the 'idx'-th key and value of a map reply. * * Returns: * - VALKEYMODULE_OK on success. * - VALKEYMODULE_ERR if idx out of range or if the reply type is wrong. * * The `key` and `value` arguments are used to return by reference, and may be * NULL if not required. */ int VM_CallReplyMapElement(ValkeyModuleCallReply *reply, size_t idx, ValkeyModuleCallReply **key, ValkeyModuleCallReply **val) { if (callReplyGetMapElement(reply, idx, key, val) == C_OK) { return VALKEYMODULE_OK; } return VALKEYMODULE_ERR; } /* Return the attribute of the given reply, or NULL if no attribute exists. */ ValkeyModuleCallReply *VM_CallReplyAttribute(ValkeyModuleCallReply *reply) { return callReplyGetAttribute(reply); } /* Retrieve the 'idx'-th key and value of an attribute reply. * * Returns: * - VALKEYMODULE_OK on success. * - VALKEYMODULE_ERR if idx out of range or if the reply type is wrong. * * The `key` and `value` arguments are used to return by reference, and may be * NULL if not required. */ int VM_CallReplyAttributeElement(ValkeyModuleCallReply *reply, size_t idx, ValkeyModuleCallReply **key, ValkeyModuleCallReply **val) { if (callReplyGetAttributeElement(reply, idx, key, val) == C_OK) { return VALKEYMODULE_OK; } return VALKEYMODULE_ERR; } /* Set unblock handler (callback and private data) on the given promise ValkeyModuleCallReply. * The given reply must be of promise type (VALKEYMODULE_REPLY_PROMISE). */ void VM_CallReplyPromiseSetUnblockHandler(ValkeyModuleCallReply *reply, ValkeyModuleOnUnblocked on_unblock, void *private_data) { ValkeyModuleAsyncRMCallPromise *promise = callReplyGetPrivateData(reply); promise->on_unblocked = on_unblock; promise->private_data = private_data; } /* Abort the execution of a given promise ValkeyModuleCallReply. * return REDMODULE_OK in case the abort was done successfully and VALKEYMODULE_ERR * if its not possible to abort the execution (execution already finished). * In case the execution was aborted (REDMODULE_OK was returned), the private_data out parameter * will be set with the value of the private data that was given on 'VM_CallReplyPromiseSetUnblockHandler' * so the caller will be able to release the private data. * * If the execution was aborted successfully, it is promised that the unblock handler will not be called. * That said, it is possible that the abort operation will successes but the operation will still continue. * This can happened if, for example, a module implements some blocking command and does not respect the * disconnect callback. For server-provided commands this can not happened.*/ int VM_CallReplyPromiseAbort(ValkeyModuleCallReply *reply, void **private_data) { ValkeyModuleAsyncRMCallPromise *promise = callReplyGetPrivateData(reply); if (!promise->c) return VALKEYMODULE_ERR; /* Promise can not be aborted, either already aborted or already finished. */ if (!(promise->c->flag.blocked)) return VALKEYMODULE_ERR; /* Client is not blocked anymore, can not abort it. */ /* Client is still blocked, remove it from any blocking state and release it. */ if (private_data) *private_data = promise->private_data; promise->private_data = NULL; promise->on_unblocked = NULL; unblockClient(promise->c, 0); moduleReleaseTempClient(promise->c); return VALKEYMODULE_OK; } /* Return the pointer and length of a string or error reply. */ const char *VM_CallReplyStringPtr(ValkeyModuleCallReply *reply, size_t *len) { size_t private_len; if (!len) len = &private_len; return callReplyGetString(reply, len); } /* Return a new string object from a call reply of type string, error or * integer. Otherwise (wrong reply type) return NULL. */ ValkeyModuleString *VM_CreateStringFromCallReply(ValkeyModuleCallReply *reply) { ValkeyModuleCtx *ctx = callReplyGetPrivateData(reply); size_t len; const char *str; switch (callReplyType(reply)) { case VALKEYMODULE_REPLY_STRING: case VALKEYMODULE_REPLY_ERROR: str = callReplyGetString(reply, &len); return VM_CreateString(ctx, str, len); case VALKEYMODULE_REPLY_INTEGER: { char buf[64]; int len = ll2string(buf, sizeof(buf), callReplyGetLongLong(reply)); return VM_CreateString(ctx, buf, len); } default: return NULL; } } /* Modifies the user that VM_Call will use (e.g. for ACL checks) */ void VM_SetContextUser(ValkeyModuleCtx *ctx, const ValkeyModuleUser *user) { ctx->user = user; } /* Returns an array of robj pointers, by parsing the format specifier "fmt" as described for * the VM_Call(), VM_Replicate() and other module APIs. Populates *argcp with the number of * items (which equals to the length of the allocated argv). * * The integer pointed by 'flags' is populated with flags according * to special modifiers in "fmt". * * "!" -> VALKEYMODULE_ARGV_REPLICATE * "A" -> VALKEYMODULE_ARGV_NO_AOF * "R" -> VALKEYMODULE_ARGV_NO_REPLICAS * "3" -> VALKEYMODULE_ARGV_RESP_3 * "0" -> VALKEYMODULE_ARGV_RESP_AUTO * "C" -> VALKEYMODULE_ARGV_RUN_AS_USER * "M" -> VALKEYMODULE_ARGV_RESPECT_DENY_OOM * "K" -> VALKEYMODULE_ARGV_ALLOW_BLOCK * * On error (format specifier error) NULL is returned and nothing is * allocated. On success the argument vector is returned. */ robj **moduleCreateArgvFromUserFormat(const char *cmdname, const char *fmt, int *argcp, int *flags, va_list ap) { int argc = 0, argv_size, j; robj **argv = NULL; /* As a first guess to avoid useless reallocations, size argv to * hold one argument for each char specifier in 'fmt'. */ argv_size = strlen(fmt) + 1; /* +1 because of the command name. */ argv = zrealloc(argv, sizeof(robj *) * argv_size); /* Build the arguments vector based on the format specifier. */ argv[0] = createStringObject(cmdname, strlen(cmdname)); argc++; /* Create the client and dispatch the command. */ const char *p = fmt; while (*p) { if (*p == 'c') { char *cstr = va_arg(ap, char *); argv[argc++] = createStringObject(cstr, strlen(cstr)); } else if (*p == 's') { robj *obj = va_arg(ap, void *); if (obj->refcount == OBJ_STATIC_REFCOUNT) obj = createStringObject(obj->ptr, sdslen(obj->ptr)); else incrRefCount(obj); argv[argc++] = obj; } else if (*p == 'b') { char *buf = va_arg(ap, char *); size_t len = va_arg(ap, size_t); argv[argc++] = createStringObject(buf, len); } else if (*p == 'l') { long long ll = va_arg(ap, long long); argv[argc++] = createStringObjectFromLongLongWithSds(ll); } else if (*p == 'v') { /* A vector of strings */ robj **v = va_arg(ap, void *); size_t vlen = va_arg(ap, size_t); /* We need to grow argv to hold the vector's elements. * We resize by vector_len-1 elements, because we held * one element in argv for the vector already */ argv_size += vlen - 1; argv = zrealloc(argv, sizeof(robj *) * argv_size); size_t i = 0; for (i = 0; i < vlen; i++) { incrRefCount(v[i]); argv[argc++] = v[i]; } } else if (*p == '!') { if (flags) (*flags) |= VALKEYMODULE_ARGV_REPLICATE; } else if (*p == 'A') { if (flags) (*flags) |= VALKEYMODULE_ARGV_NO_AOF; } else if (*p == 'R') { if (flags) (*flags) |= VALKEYMODULE_ARGV_NO_REPLICAS; } else if (*p == '3') { if (flags) (*flags) |= VALKEYMODULE_ARGV_RESP_3; } else if (*p == '0') { if (flags) (*flags) |= VALKEYMODULE_ARGV_RESP_AUTO; } else if (*p == 'C') { if (flags) (*flags) |= VALKEYMODULE_ARGV_RUN_AS_USER; } else if (*p == 'S') { if (flags) (*flags) |= VALKEYMODULE_ARGV_SCRIPT_MODE; } else if (*p == 'W') { if (flags) (*flags) |= VALKEYMODULE_ARGV_NO_WRITES; } else if (*p == 'M') { if (flags) (*flags) |= VALKEYMODULE_ARGV_RESPECT_DENY_OOM; } else if (*p == 'E') { if (flags) (*flags) |= VALKEYMODULE_ARGV_CALL_REPLIES_AS_ERRORS; } else if (*p == 'D') { if (flags) (*flags) |= (VALKEYMODULE_ARGV_DRY_RUN | VALKEYMODULE_ARGV_CALL_REPLIES_AS_ERRORS); } else if (*p == 'K') { if (flags) (*flags) |= VALKEYMODULE_ARGV_ALLOW_BLOCK; } else { goto fmterr; } p++; } if (argcp) *argcp = argc; return argv; fmterr: for (j = 0; j < argc; j++) decrRefCount(argv[j]); zfree(argv); return NULL; } /* Exported API to call any command from modules. * * * **cmdname**: The command to call. * * **fmt**: A format specifier string for the command's arguments. Each * of the arguments should be specified by a valid type specification. The * format specifier can also contain the modifiers `!`, `A`, `3` and `R` which * don't have a corresponding argument. * * * `b` -- The argument is a buffer and is immediately followed by another * argument that is the buffer's length. * * `c` -- The argument is a pointer to a plain C string (null-terminated). * * `l` -- The argument is a `long long` integer. * * `s` -- The argument is a ValkeyModuleString. * * `v` -- The argument(s) is a vector of ValkeyModuleString. * * `!` -- Sends the command and its arguments to replicas and AOF. * * `A` -- Suppress AOF propagation, send only to replicas (requires `!`). * * `R` -- Suppress replicas propagation, send only to AOF (requires `!`). * * `3` -- Return a RESP3 reply. This will change the command reply. * e.g., HGETALL returns a map instead of a flat array. * * `0` -- Return the reply in auto mode, i.e. the reply format will be the * same as the client attached to the given ValkeyModuleCtx. This will * probably used when you want to pass the reply directly to the client. * * `C` -- Run a command as the user attached to the context. * User is either attached automatically via the client that directly * issued the command and created the context or via VM_SetContextUser. * If the context is not directly created by an issued command (such as a * background context and no user was set on it via VM_SetContextUser, * VM_Call will fail. * Checks if the command can be executed according to ACL rules and causes * the command to run as the determined user, so that any future user * dependent activity, such as ACL checks within scripts will proceed as * expected. * Otherwise, the command will run as the unrestricted user. * * `S` -- Run the command in a script mode, this means that it will raise * an error if a command which are not allowed inside a script * (flagged with the `deny-script` flag) is invoked (like SHUTDOWN). * In addition, on script mode, write commands are not allowed if there are * not enough good replicas (as configured with `min-replicas-to-write`) * or when the server is unable to persist to the disk. * * `W` -- Do not allow to run any write command (flagged with the `write` flag). * * `M` -- Do not allow `deny-oom` flagged commands when over the memory limit. * * `E` -- Return error as ValkeyModuleCallReply. If there is an error before * invoking the command, the error is returned using errno mechanism. * This flag allows to get the error also as an error CallReply with * relevant error message. * * 'D' -- A "Dry Run" mode. Return before executing the underlying call(). * If everything succeeded, it will return with a NULL, otherwise it will * return with a CallReply object denoting the error, as if it was called with * the 'E' code. * * 'K' -- Allow running blocking commands. If enabled and the command gets blocked, a * special VALKEYMODULE_REPLY_PROMISE will be returned. This reply type * indicates that the command was blocked and the reply will be given asynchronously. * The module can use this reply object to set a handler which will be called when * the command gets unblocked using ValkeyModule_CallReplyPromiseSetUnblockHandler. * The handler must be set immediately after the command invocation (without releasing * the lock in between). If the handler is not set, the blocking command will * still continue its execution but the reply will be ignored (fire and forget), * notice that this is dangerous in case of role change, as explained below. * The module can use ValkeyModule_CallReplyPromiseAbort to abort the command invocation * if it was not yet finished (see ValkeyModule_CallReplyPromiseAbort documentation for more * details). It is also the module's responsibility to abort the execution on role change, either by using * server event (to get notified when the instance becomes a replica) or relying on the disconnect * callback of the original client. Failing to do so can result in a write operation on a replica. * Unlike other call replies, promise call reply **must** be freed while the GIL is locked. * Notice that on unblocking, the only promise is that the unblock handler will be called, * If the blocking VM_Call caused the module to also block some real client (using VM_BlockClient), * it is the module responsibility to unblock this client on the unblock handler. * On the unblock handler it is only allowed to perform the following: * * Calling additional commands using VM_Call * * Open keys using VM_OpenKey * * Replicate data to the replica or AOF * * Specifically, it is not allowed to call any module API which are client related such as: * * VM_Reply* API's * * VM_BlockClient * * VM_GetCurrentUserName * * * **...**: The actual arguments to the command. * * On success a ValkeyModuleCallReply object is returned, otherwise * NULL is returned and errno is set to the following values: * * * EBADF: wrong format specifier. * * EINVAL: wrong command arity. * * ENOENT: command does not exist. * * EPERM: operation in Cluster instance with key in non local slot. * * EROFS: operation in Cluster instance when a write command is sent * in a readonly state. * * ENETDOWN: operation in Cluster instance when cluster is down. * * ENOTSUP: No ACL user for the specified module context * * EACCES: Command cannot be executed, according to ACL rules * * ENOSPC: Write or deny-oom command is not allowed * * ESPIPE: Command not allowed on script mode * * Example code fragment: * * reply = ValkeyModule_Call(ctx,"INCRBY","sc",argv[1],"10"); * if (ValkeyModule_CallReplyType(reply) == VALKEYMODULE_REPLY_INTEGER) { * long long myval = ValkeyModule_CallReplyInteger(reply); * // Do something with myval. * } * * This API is documented here: https://valkey.io/topics/modules-intro */ ValkeyModuleCallReply *VM_Call(ValkeyModuleCtx *ctx, const char *cmdname, const char *fmt, ...) { client *c = NULL; robj **argv = NULL; int argc = 0, flags = 0; va_list ap; ValkeyModuleCallReply *reply = NULL; int replicate = 0; /* Replicate this command? */ int error_as_call_replies = 0; /* return errors as ValkeyModuleCallReply object */ uint64_t cmd_flags; /* Handle arguments. */ va_start(ap, fmt); argv = moduleCreateArgvFromUserFormat(cmdname, fmt, &argc, &flags, ap); replicate = flags & VALKEYMODULE_ARGV_REPLICATE; error_as_call_replies = flags & VALKEYMODULE_ARGV_CALL_REPLIES_AS_ERRORS; va_end(ap); c = moduleAllocTempClient(); if (!(flags & VALKEYMODULE_ARGV_ALLOW_BLOCK)) { /* We do not want to allow block, the module do not expect it */ c->flag.deny_blocking = 1; } c->db = ctx->client->db; c->argv = argv; /* We have to assign argv_len, which is equal to argc in that case (VM_Call) * because we may be calling a command that uses rewriteClientCommandArgument */ c->argc = c->argv_len = argc; c->resp = 2; if (flags & VALKEYMODULE_ARGV_RESP_3) { c->resp = 3; } else if (flags & VALKEYMODULE_ARGV_RESP_AUTO) { /* Auto mode means to take the same protocol as the ctx client. */ c->resp = ctx->client->resp; } if (ctx->module) ctx->module->in_call++; user *user = NULL; if (flags & VALKEYMODULE_ARGV_RUN_AS_USER) { user = ctx->user ? ctx->user->user : ctx->client->user; if (!user) { errno = ENOTSUP; if (error_as_call_replies) { sds msg = sdsnew("cannot run as user, no user directly attached to context or context's client"); reply = callReplyCreateError(msg, ctx); } goto cleanup; } c->user = user; } /* We handle the above format error only when the client is setup so that * we can free it normally. */ if (argv == NULL) { /* We do not return a call reply here this is an error that should only * be catch by the module indicating wrong fmt was given, the module should * handle this error and decide how to continue. It is not an error that * should be propagated to the user. */ errno = EBADF; goto cleanup; } /* Call command filters */ moduleCallCommandFilters(c); /* Lookup command now, after filters had a chance to make modifications * if necessary. */ c->cmd = c->lastcmd = c->realcmd = lookupCommand(c->argv, c->argc); sds err; if (!commandCheckExistence(c, error_as_call_replies ? &err : NULL)) { errno = ENOENT; if (error_as_call_replies) reply = callReplyCreateError(err, ctx); goto cleanup; } if (!commandCheckArity(c->cmd, c->argc, error_as_call_replies ? &err : NULL)) { errno = EINVAL; if (error_as_call_replies) reply = callReplyCreateError(err, ctx); goto cleanup; } cmd_flags = getCommandFlags(c); if (flags & VALKEYMODULE_ARGV_SCRIPT_MODE) { /* Basically on script mode we want to only allow commands that can * be executed on scripts (CMD_NOSCRIPT is not set on the command flags) */ if (cmd_flags & CMD_NOSCRIPT) { errno = ESPIPE; if (error_as_call_replies) { sds msg = sdscatfmt(sdsempty(), "command '%S' is not allowed on script mode", c->cmd->fullname); reply = callReplyCreateError(msg, ctx); } goto cleanup; } } if (flags & VALKEYMODULE_ARGV_RESPECT_DENY_OOM && server.maxmemory) { if (cmd_flags & CMD_DENYOOM) { int oom_state; if (ctx->flags & VALKEYMODULE_CTX_THREAD_SAFE) { /* On background thread we can not count on server.pre_command_oom_state. * Because it is only set on the main thread, in such case we will check * the actual memory usage. */ oom_state = (getMaxmemoryState(NULL, NULL, NULL, NULL) == C_ERR); } else { oom_state = server.pre_command_oom_state; } if (oom_state) { errno = ENOSPC; if (error_as_call_replies) { sds msg = sdsdup(shared.oomerr->ptr); reply = callReplyCreateError(msg, ctx); } goto cleanup; } } } else { /* if we aren't OOM checking in VM_Call, we want further executions from this client to also not fail on OOM */ c->flag.allow_oom = 1; } if (flags & VALKEYMODULE_ARGV_NO_WRITES) { if (cmd_flags & CMD_WRITE) { errno = ENOSPC; if (error_as_call_replies) { sds msg = sdscatfmt(sdsempty(), "Write command '%S' was " "called while write is not allowed.", c->cmd->fullname); reply = callReplyCreateError(msg, ctx); } goto cleanup; } } /* Script mode tests */ if (flags & VALKEYMODULE_ARGV_SCRIPT_MODE) { if (cmd_flags & CMD_WRITE) { /* on script mode, if a command is a write command, * We will not run it if we encounter disk error * or we do not have enough replicas */ if (!checkGoodReplicasStatus()) { errno = ESPIPE; if (error_as_call_replies) { sds msg = sdsdup(shared.noreplicaserr->ptr); reply = callReplyCreateError(msg, ctx); } goto cleanup; } int deny_write_type = writeCommandsDeniedByDiskError(); int obey_client = (server.current_client && mustObeyClient(server.current_client)); if (deny_write_type != DISK_ERROR_TYPE_NONE && !obey_client) { errno = ESPIPE; if (error_as_call_replies) { sds msg = writeCommandsGetDiskErrorMessage(deny_write_type); reply = callReplyCreateError(msg, ctx); } goto cleanup; } if (server.primary_host && server.repl_replica_ro && !obey_client) { errno = ESPIPE; if (error_as_call_replies) { sds msg = sdsdup(shared.roreplicaerr->ptr); reply = callReplyCreateError(msg, ctx); } goto cleanup; } } if (server.primary_host && server.repl_state != REPL_STATE_CONNECTED && server.repl_serve_stale_data == 0 && !(cmd_flags & CMD_STALE)) { errno = ESPIPE; if (error_as_call_replies) { sds msg = sdsdup(shared.primarydownerr->ptr); reply = callReplyCreateError(msg, ctx); } goto cleanup; } } /* Check if the user can run this command according to the current * ACLs. * * If VM_SetContextUser has set a user, that user is used, otherwise * use the attached client's user. If there is no attached client user and no manually * set user, an error will be returned */ if (flags & VALKEYMODULE_ARGV_RUN_AS_USER) { int acl_errpos; int acl_retval; acl_retval = ACLCheckAllUserCommandPerm(user, c->cmd, c->argv, c->argc, &acl_errpos); if (acl_retval != ACL_OK) { sds object = (acl_retval == ACL_DENIED_CMD) ? sdsdup(c->cmd->fullname) : sdsdup(c->argv[acl_errpos]->ptr); addACLLogEntry(ctx->client, acl_retval, ACL_LOG_CTX_MODULE, -1, c->user->name, object); if (error_as_call_replies) { /* verbosity should be same as processCommand() in server.c */ sds acl_msg = getAclErrorMessage(acl_retval, c->user, c->cmd, c->argv[acl_errpos]->ptr, 0); sds msg = sdscatfmt(sdsempty(), "-NOPERM %S\r\n", acl_msg); sdsfree(acl_msg); reply = callReplyCreateError(msg, ctx); } errno = EACCES; goto cleanup; } } /* If this is a Cluster node, we need to make sure the module is not * trying to access non-local keys, with the exception of commands * received from our primary. */ if (server.cluster_enabled && !mustObeyClient(ctx->client)) { int error_code; /* Duplicate relevant flags in the module client. */ c->flag.readonly = ctx->client->flag.readonly; c->flag.asking = ctx->client->flag.asking; if (getNodeByQuery(c, c->cmd, c->argv, c->argc, NULL, &error_code) != getMyClusterNode()) { sds msg = NULL; if (error_code == CLUSTER_REDIR_DOWN_RO_STATE) { if (error_as_call_replies) { msg = sdscatfmt(sdsempty(), "Can not execute a write command '%S' while the cluster is down and readonly", c->cmd->fullname); } errno = EROFS; } else if (error_code == CLUSTER_REDIR_DOWN_STATE) { if (error_as_call_replies) { msg = sdscatfmt(sdsempty(), "Can not execute a command '%S' while the cluster is down", c->cmd->fullname); } errno = ENETDOWN; } else { if (error_as_call_replies) { msg = sdsnew("Attempted to access a non local key in a cluster node"); } errno = EPERM; } if (msg) { reply = callReplyCreateError(msg, ctx); } goto cleanup; } } if (flags & VALKEYMODULE_ARGV_DRY_RUN) { goto cleanup; } /* We need to use a global replication_allowed flag in order to prevent * replication of nested VM_Calls. Example: * 1. module1.foo does VM_Call of module2.bar without replication (i.e. no '!') * 2. module2.bar internally calls VM_Call of INCR with '!' * 3. at the end of module1.foo we call VM_ReplicateVerbatim * We want the replica/AOF to see only module1.foo and not the INCR from module2.bar */ int prev_replication_allowed = server.replication_allowed; server.replication_allowed = replicate && server.replication_allowed; /* Run the command */ int call_flags = CMD_CALL_FROM_MODULE; if (replicate) { if (!(flags & VALKEYMODULE_ARGV_NO_AOF)) call_flags |= CMD_CALL_PROPAGATE_AOF; if (!(flags & VALKEYMODULE_ARGV_NO_REPLICAS)) call_flags |= CMD_CALL_PROPAGATE_REPL; } call(c, call_flags); server.replication_allowed = prev_replication_allowed; if (c->flag.blocked) { serverAssert(flags & VALKEYMODULE_ARGV_ALLOW_BLOCK); serverAssert(ctx->module); ValkeyModuleAsyncRMCallPromise *promise = zmalloc(sizeof(ValkeyModuleAsyncRMCallPromise)); *promise = (ValkeyModuleAsyncRMCallPromise){ /* We start with ref_count value of 2 because this object is held * by the promise CallReply and the fake client that was used to execute the command. */ .ref_count = 2, .module = ctx->module, .on_unblocked = NULL, .private_data = NULL, .c = c, .ctx = (ctx->flags & VALKEYMODULE_CTX_AUTO_MEMORY) ? ctx : NULL, }; reply = callReplyCreatePromise(promise); c->bstate->async_rm_call_handle = promise; if (!(call_flags & CMD_CALL_PROPAGATE_AOF)) { /* No need for AOF propagation, set the relevant flags of the client */ c->flag.module_prevent_aof_prop = 1; } if (!(call_flags & CMD_CALL_PROPAGATE_REPL)) { /* No need for replication propagation, set the relevant flags of the client */ c->flag.module_prevent_repl_prop = 1; } c = NULL; /* Make sure not to free the client */ } else { reply = moduleParseReply(c, (ctx->flags & VALKEYMODULE_CTX_AUTO_MEMORY) ? ctx : NULL); } cleanup: if (reply) autoMemoryAdd(ctx, VALKEYMODULE_AM_REPLY, reply); if (ctx->module) ctx->module->in_call--; if (c) moduleReleaseTempClient(c); return reply; } /* Return a pointer, and a length, to the protocol returned by the command * that returned the reply object. */ const char *VM_CallReplyProto(ValkeyModuleCallReply *reply, size_t *len) { return callReplyGetProto(reply, len); } /* -------------------------------------------------------------------------- * ## Modules data types * * When String DMA or using existing data structures is not enough, it is * possible to create new data types from scratch. * The module must provide a set of callbacks for handling the * new values exported (for example in order to provide RDB saving/loading, * AOF rewrite, and so forth). In this section we define this API. * -------------------------------------------------------------------------- */ /* Turn a 9 chars name in the specified charset and a 10 bit encver into * a single 64 bit unsigned integer that represents this exact module name * and version. This final number is called a "type ID" and is used when * writing module exported values to RDB files, in order to re-associate the * value to the right module to load them during RDB loading. * * If the string is not of the right length or the charset is wrong, or * if encver is outside the unsigned 10 bit integer range, 0 is returned, * otherwise the function returns the right type ID. * * The resulting 64 bit integer is composed as follows: * * (high order bits) 6|6|6|6|6|6|6|6|6|10 (low order bits) * * The first 6 bits value is the first character, name[0], while the last * 6 bits value, immediately before the 10 bits integer, is name[8]. * The last 10 bits are the encoding version. * * Note that a name and encver combo of "AAAAAAAAA" and 0, will produce * zero as return value, that is the same we use to signal errors, thus * this combination is invalid, and also useless since type names should * try to be vary to avoid collisions. */ const char *ModuleTypeNameCharSet = "ABCDEFGHIJKLMNOPQRSTUVWXYZ" "abcdefghijklmnopqrstuvwxyz" "0123456789-_"; uint64_t moduleTypeEncodeId(const char *name, int encver) { /* We use 64 symbols so that we can map each character into 6 bits * of the final output. */ const char *cset = ModuleTypeNameCharSet; if (strlen(name) != 9) return 0; if (encver < 0 || encver > 1023) return 0; uint64_t id = 0; for (int j = 0; j < 9; j++) { char *p = strchr(cset, name[j]); if (!p) return 0; unsigned long pos = p - cset; id = (id << 6) | pos; } id = (id << 10) | encver; return id; } /* Search, in the list of exported data types of all the modules registered, * a type with the same name as the one given. Returns the moduleType * structure pointer if such a module is found, or NULL otherwise. */ moduleType *moduleTypeLookupModuleByNameInternal(const char *name, int ignore_case) { dictIterator *di = dictGetIterator(modules); dictEntry *de; while ((de = dictNext(di)) != NULL) { struct ValkeyModule *module = dictGetVal(de); listIter li; listNode *ln; listRewind(module->types, &li); while ((ln = listNext(&li))) { moduleType *mt = ln->value; if ((!ignore_case && memcmp(name, mt->name, sizeof(mt->name)) == 0) || (ignore_case && !strcasecmp(name, mt->name))) { dictReleaseIterator(di); return mt; } } } dictReleaseIterator(di); return NULL; } /* Search all registered modules by name, and name is case sensitive */ moduleType *moduleTypeLookupModuleByName(const char *name) { return moduleTypeLookupModuleByNameInternal(name, 0); } /* Search all registered modules by name, but case insensitive */ moduleType *moduleTypeLookupModuleByNameIgnoreCase(const char *name) { return moduleTypeLookupModuleByNameInternal(name, 1); } /* Lookup a module by ID, with caching. This function is used during RDB * loading. Modules exporting data types should never be able to unload, so * our cache does not need to expire. */ #define MODULE_LOOKUP_CACHE_SIZE 3 moduleType *moduleTypeLookupModuleByID(uint64_t id) { static struct { uint64_t id; moduleType *mt; } cache[MODULE_LOOKUP_CACHE_SIZE]; /* Search in cache to start. */ int j; for (j = 0; j < MODULE_LOOKUP_CACHE_SIZE && cache[j].mt != NULL; j++) if (cache[j].id == id) return cache[j].mt; /* Slow module by module lookup. */ moduleType *mt = NULL; dictIterator *di = dictGetIterator(modules); dictEntry *de; while ((de = dictNext(di)) != NULL && mt == NULL) { struct ValkeyModule *module = dictGetVal(de); listIter li; listNode *ln; listRewind(module->types, &li); while ((ln = listNext(&li))) { moduleType *this_mt = ln->value; /* Compare only the 54 bit module identifier and not the * encoding version. */ if (this_mt->id >> 10 == id >> 10) { mt = this_mt; break; } } } dictReleaseIterator(di); /* Add to cache if possible. */ if (mt && j < MODULE_LOOKUP_CACHE_SIZE) { cache[j].id = id; cache[j].mt = mt; } return mt; } /* Turn an (unresolved) module ID into a type name, to show the user an * error when RDB files contain module data we can't load. * The buffer pointed by 'name' must be 10 bytes at least. The function will * fill it with a null terminated module name. */ void moduleTypeNameByID(char *name, uint64_t moduleid) { const char *cset = ModuleTypeNameCharSet; name[9] = '\0'; char *p = name + 8; moduleid >>= 10; for (int j = 0; j < 9; j++) { *p-- = cset[moduleid & 63]; moduleid >>= 6; } } /* Return the name of the module that owns the specified moduleType. */ const char *moduleTypeModuleName(moduleType *mt) { if (!mt || !mt->module) return NULL; return mt->module->name; } /* Return the module name from a module command */ const char *moduleNameFromCommand(struct serverCommand *cmd) { serverAssert(cmd->proc == ValkeyModuleCommandDispatcher); ValkeyModuleCommand *cp = cmd->module_cmd; return cp->module->name; } /* Create a copy of a module type value using the copy callback. If failed * or not supported, produce an error reply and return NULL. */ robj *moduleTypeDupOrReply(client *c, robj *fromkey, robj *tokey, int todb, robj *value) { moduleValue *mv = value->ptr; moduleType *mt = mv->type; if (!mt->copy && !mt->copy2) { addReplyError(c, "not supported for this module key"); return NULL; } void *newval = NULL; if (mt->copy2 != NULL) { ValkeyModuleKeyOptCtx ctx = {fromkey, tokey, c->db->id, todb}; newval = mt->copy2(&ctx, mv->value); } else { newval = mt->copy(fromkey, tokey, mv->value); } if (!newval) { addReplyError(c, "module key failed to copy"); return NULL; } return createModuleObject(mt, newval); } /* Register a new data type exported by the module. The parameters are the * following. Please for in depth documentation check the modules API * documentation, especially https://valkey.io/topics/modules-native-types. * * * **name**: A 9 characters data type name that MUST be unique in the * Modules ecosystem. Be creative... and there will be no collisions. Use * the charset A-Z a-z 9-0, plus the two "-_" characters. A good * idea is to use, for example `-`. For example * "tree-AntZ" may mean "Tree data structure by @antirez". To use both * lower case and upper case letters helps in order to prevent collisions. * * **encver**: Encoding version, which is, the version of the serialization * that a module used in order to persist data. As long as the "name" * matches, the RDB loading will be dispatched to the type callbacks * whatever 'encver' is used, however the module can understand if * the encoding it must load are of an older version of the module. * For example the module "tree-AntZ" initially used encver=0. Later * after an upgrade, it started to serialize data in a different format * and to register the type with encver=1. However this module may * still load old data produced by an older version if the rdb_load * callback is able to check the encver value and act accordingly. * The encver must be a positive value between 0 and 1023. * * * **typemethods_ptr** is a pointer to a ValkeyModuleTypeMethods structure * that should be populated with the methods callbacks and structure * version, like in the following example: * * ValkeyModuleTypeMethods tm = { * .version = VALKEYMODULE_TYPE_METHOD_VERSION, * .rdb_load = myType_RDBLoadCallBack, * .rdb_save = myType_RDBSaveCallBack, * .aof_rewrite = myType_AOFRewriteCallBack, * .free = myType_FreeCallBack, * * // Optional fields * .digest = myType_DigestCallBack, * .mem_usage = myType_MemUsageCallBack, * .aux_load = myType_AuxRDBLoadCallBack, * .aux_save = myType_AuxRDBSaveCallBack, * .free_effort = myType_FreeEffortCallBack, * .unlink = myType_UnlinkCallBack, * .copy = myType_CopyCallback, * .defrag = myType_DefragCallback * * // Enhanced optional fields * .mem_usage2 = myType_MemUsageCallBack2, * .free_effort2 = myType_FreeEffortCallBack2, * .unlink2 = myType_UnlinkCallBack2, * .copy2 = myType_CopyCallback2, * } * * * **rdb_load**: A callback function pointer that loads data from RDB files. * * **rdb_save**: A callback function pointer that saves data to RDB files. * * **aof_rewrite**: A callback function pointer that rewrites data as commands. * * **digest**: A callback function pointer that is used for `DEBUG DIGEST`. * * **free**: A callback function pointer that can free a type value. * * **aux_save**: A callback function pointer that saves out of keyspace data to RDB files. * 'when' argument is either VALKEYMODULE_AUX_BEFORE_RDB or VALKEYMODULE_AUX_AFTER_RDB. * * **aux_load**: A callback function pointer that loads out of keyspace data from RDB files. * Similar to aux_save, returns VALKEYMODULE_OK on success, and ERR otherwise. * * **free_effort**: A callback function pointer that used to determine whether the module's * memory needs to be lazy reclaimed. The module should return the complexity involved by * freeing the value. for example: how many pointers are gonna be freed. Note that if it * returns 0, we'll always do an async free. * * **unlink**: A callback function pointer that used to notifies the module that the key has * been removed from the DB by the server, and may soon be freed by a background thread. Note that * it won't be called on FLUSHALL/FLUSHDB (both sync and async), and the module can use the * ValkeyModuleEvent_FlushDB to hook into that. * * **copy**: A callback function pointer that is used to make a copy of the specified key. * The module is expected to perform a deep copy of the specified value and return it. * In addition, hints about the names of the source and destination keys is provided. * A NULL return value is considered an error and the copy operation fails. * Note: if the target key exists and is being overwritten, the copy callback will be * called first, followed by a free callback to the value that is being replaced. * * * **defrag**: A callback function pointer that is used to request the module to defrag * a key. The module should then iterate pointers and call the relevant VM_Defrag*() * functions to defragment pointers or complex types. The module should continue * iterating as long as VM_DefragShouldStop() returns a zero value, and return a * zero value if finished or non-zero value if more work is left to be done. If more work * needs to be done, VM_DefragCursorSet() and VM_DefragCursorGet() can be used to track * this work across different calls. * Normally, the defrag mechanism invokes the callback without a time limit, so * VM_DefragShouldStop() always returns zero. The "late defrag" mechanism which has * a time limit and provides cursor support is used only for keys that are determined * to have significant internal complexity. To determine this, the defrag mechanism * uses the free_effort callback and the 'active-defrag-max-scan-fields' config directive. * NOTE: The value is passed as a `void**` and the function is expected to update the * pointer if the top-level value pointer is defragmented and consequently changes. * * * **mem_usage2**: Similar to `mem_usage`, but provides the `ValkeyModuleKeyOptCtx` parameter * so that meta information such as key name and db id can be obtained, and * the `sample_size` for size estimation (see MEMORY USAGE command). * * **free_effort2**: Similar to `free_effort`, but provides the `ValkeyModuleKeyOptCtx` parameter * so that meta information such as key name and db id can be obtained. * * **unlink2**: Similar to `unlink`, but provides the `ValkeyModuleKeyOptCtx` parameter * so that meta information such as key name and db id can be obtained. * * **copy2**: Similar to `copy`, but provides the `ValkeyModuleKeyOptCtx` parameter * so that meta information such as key names and db ids can be obtained. * * **aux_save2**: Similar to `aux_save`, but with small semantic change, if the module * saves nothing on this callback then no data about this aux field will be written to the * RDB and it will be possible to load the RDB even if the module is not loaded. * * Note: the module name "AAAAAAAAA" is reserved and produces an error, it * happens to be pretty lame as well. * * If ValkeyModule_CreateDataType() is called outside of ValkeyModule_OnLoad() function, * there is already a module registering a type with the same name, * or if the module name or encver is invalid, NULL is returned. * Otherwise the new type is registered into the server, and a reference of * type ValkeyModuleType is returned: the caller of the function should store * this reference into a global variable to make future use of it in the * modules type API, since a single module may register multiple types. * Example code fragment: * * static ValkeyModuleType *BalancedTreeType; * * int ValkeyModule_OnLoad(ValkeyModuleCtx *ctx) { * // some code here ... * BalancedTreeType = VM_CreateDataType(...); * } */ moduleType *VM_CreateDataType(ValkeyModuleCtx *ctx, const char *name, int encver, void *typemethods_ptr) { if (!ctx->module->onload) return NULL; uint64_t id = moduleTypeEncodeId(name, encver); if (id == 0) return NULL; if (moduleTypeLookupModuleByName(name) != NULL) return NULL; long typemethods_version = ((long *)typemethods_ptr)[0]; if (typemethods_version == 0) return NULL; struct typemethods { uint64_t version; moduleTypeLoadFunc rdb_load; moduleTypeSaveFunc rdb_save; moduleTypeRewriteFunc aof_rewrite; moduleTypeMemUsageFunc mem_usage; moduleTypeDigestFunc digest; moduleTypeFreeFunc free; struct { moduleTypeAuxLoadFunc aux_load; moduleTypeAuxSaveFunc aux_save; int aux_save_triggers; } v2; struct { moduleTypeFreeEffortFunc free_effort; moduleTypeUnlinkFunc unlink; moduleTypeCopyFunc copy; moduleTypeDefragFunc defrag; } v3; struct { moduleTypeMemUsageFunc2 mem_usage2; moduleTypeFreeEffortFunc2 free_effort2; moduleTypeUnlinkFunc2 unlink2; moduleTypeCopyFunc2 copy2; } v4; struct { moduleTypeAuxSaveFunc aux_save2; } v5; } *tms = (struct typemethods *)typemethods_ptr; moduleType *mt = zcalloc(sizeof(*mt)); mt->id = id; mt->module = ctx->module; mt->rdb_load = tms->rdb_load; mt->rdb_save = tms->rdb_save; mt->aof_rewrite = tms->aof_rewrite; mt->mem_usage = tms->mem_usage; mt->digest = tms->digest; mt->free = tms->free; if (tms->version >= 2) { mt->aux_load = tms->v2.aux_load; mt->aux_save = tms->v2.aux_save; mt->aux_save_triggers = tms->v2.aux_save_triggers; } if (tms->version >= 3) { mt->free_effort = tms->v3.free_effort; mt->unlink = tms->v3.unlink; mt->copy = tms->v3.copy; mt->defrag = tms->v3.defrag; } if (tms->version >= 4) { mt->mem_usage2 = tms->v4.mem_usage2; mt->unlink2 = tms->v4.unlink2; mt->free_effort2 = tms->v4.free_effort2; mt->copy2 = tms->v4.copy2; } if (tms->version >= 5) { mt->aux_save2 = tms->v5.aux_save2; } memcpy(mt->name, name, sizeof(mt->name)); listAddNodeTail(ctx->module->types, mt); return mt; } /* If the key is open for writing, set the specified module type object * as the value of the key, deleting the old value if any. * On success VALKEYMODULE_OK is returned. If the key is not open for * writing or there is an active iterator, VALKEYMODULE_ERR is returned. */ int VM_ModuleTypeSetValue(ValkeyModuleKey *key, moduleType *mt, void *value) { if (!(key->mode & VALKEYMODULE_WRITE) || key->iter) return VALKEYMODULE_ERR; VM_DeleteKey(key); robj *o = createModuleObject(mt, value); setKey(key->ctx->client, key->db, key->key, &o, SETKEY_NO_SIGNAL); key->value = o; return VALKEYMODULE_OK; } /* Assuming ValkeyModule_KeyType() returned VALKEYMODULE_KEYTYPE_MODULE on * the key, returns the module type pointer of the value stored at key. * * If the key is NULL, is not associated with a module type, or is empty, * then NULL is returned instead. */ moduleType *VM_ModuleTypeGetType(ValkeyModuleKey *key) { if (key == NULL || key->value == NULL || VM_KeyType(key) != VALKEYMODULE_KEYTYPE_MODULE) return NULL; moduleValue *mv = key->value->ptr; return mv->type; } /* Assuming ValkeyModule_KeyType() returned VALKEYMODULE_KEYTYPE_MODULE on * the key, returns the module type low-level value stored at key, as * it was set by the user via ValkeyModule_ModuleTypeSetValue(). * * If the key is NULL, is not associated with a module type, or is empty, * then NULL is returned instead. */ void *VM_ModuleTypeGetValue(ValkeyModuleKey *key) { if (key == NULL || key->value == NULL || VM_KeyType(key) != VALKEYMODULE_KEYTYPE_MODULE) return NULL; moduleValue *mv = key->value->ptr; return mv->value; } /* -------------------------------------------------------------------------- * ## RDB loading and saving functions * -------------------------------------------------------------------------- */ /* Called when there is a load error in the context of a module. On some * modules this cannot be recovered, but if the module declared capability * to handle errors, we'll raise a flag rather than exiting. */ void moduleRDBLoadError(ValkeyModuleIO *io) { if (io->type->module->options & VALKEYMODULE_OPTIONS_HANDLE_IO_ERRORS) { io->error = 1; return; } serverPanic("Error loading data from RDB (short read or EOF). " "Read performed by module '%s' about type '%s' " "after reading '%llu' bytes of a value " "for key named: '%s'.", io->type->module->name, io->type->name, (unsigned long long)io->bytes, io->key ? (char *)io->key->ptr : "(null)"); } /* Returns 0 if there's at least one registered data type that did not declare * VALKEYMODULE_OPTIONS_HANDLE_IO_ERRORS, in which case diskless loading should * be avoided since it could cause data loss. */ int moduleAllDatatypesHandleErrors(void) { dictIterator *di = dictGetIterator(modules); dictEntry *de; while ((de = dictNext(di)) != NULL) { struct ValkeyModule *module = dictGetVal(de); if (listLength(module->types) && !(module->options & VALKEYMODULE_OPTIONS_HANDLE_IO_ERRORS)) { dictReleaseIterator(di); return 0; } } dictReleaseIterator(di); return 1; } /* Returns 0 if module did not declare VALKEYMODULE_OPTIONS_HANDLE_REPL_ASYNC_LOAD, in which case * diskless async loading should be avoided because module doesn't know there can be traffic during * database full resynchronization. */ int moduleAllModulesHandleReplAsyncLoad(void) { dictIterator *di = dictGetIterator(modules); dictEntry *de; while ((de = dictNext(di)) != NULL) { struct ValkeyModule *module = dictGetVal(de); if (!(module->options & VALKEYMODULE_OPTIONS_HANDLE_REPL_ASYNC_LOAD)) { dictReleaseIterator(di); return 0; } } dictReleaseIterator(di); return 1; } /* Returns true if any previous IO API failed. * for `Load*` APIs the VALKEYMODULE_OPTIONS_HANDLE_IO_ERRORS flag must be set with * ValkeyModule_SetModuleOptions first. */ int VM_IsIOError(ValkeyModuleIO *io) { return io->error; } static int flushValkeyModuleIOBuffer(ValkeyModuleIO *io) { if (!io->pre_flush_buffer) return 0; /* We have data that must be flushed before saving the current data. * Lets flush it. */ sds pre_flush_buffer = io->pre_flush_buffer; io->pre_flush_buffer = NULL; ssize_t retval = rdbWriteRaw(io->rio, pre_flush_buffer, sdslen(pre_flush_buffer)); sdsfree(pre_flush_buffer); if (retval >= 0) io->bytes += retval; return retval; } /* Save an unsigned 64 bit value into the RDB file. This function should only * be called in the context of the rdb_save method of modules implementing new * data types. */ void VM_SaveUnsigned(ValkeyModuleIO *io, uint64_t value) { if (io->error) return; if (flushValkeyModuleIOBuffer(io) == -1) goto saveerr; /* Save opcode. */ int retval = rdbSaveLen(io->rio, RDB_MODULE_OPCODE_UINT); if (retval == -1) goto saveerr; io->bytes += retval; /* Save value. */ retval = rdbSaveLen(io->rio, value); if (retval == -1) goto saveerr; io->bytes += retval; return; saveerr: io->error = 1; } /* Load an unsigned 64 bit value from the RDB file. This function should only * be called in the context of the `rdb_load` method of modules implementing * new data types. */ uint64_t VM_LoadUnsigned(ValkeyModuleIO *io) { if (io->error) return 0; uint64_t opcode = rdbLoadLen(io->rio, NULL); if (opcode != RDB_MODULE_OPCODE_UINT) goto loaderr; uint64_t value; int retval = rdbLoadLenByRef(io->rio, NULL, &value); if (retval == -1) goto loaderr; return value; loaderr: moduleRDBLoadError(io); return 0; } /* Like ValkeyModule_SaveUnsigned() but for signed 64 bit values. */ void VM_SaveSigned(ValkeyModuleIO *io, int64_t value) { union { uint64_t u; int64_t i; } conv; conv.i = value; VM_SaveUnsigned(io, conv.u); } /* Like ValkeyModule_LoadUnsigned() but for signed 64 bit values. */ int64_t VM_LoadSigned(ValkeyModuleIO *io) { union { uint64_t u; int64_t i; } conv; conv.u = VM_LoadUnsigned(io); return conv.i; } /* In the context of the rdb_save method of a module type, saves a * string into the RDB file taking as input a ValkeyModuleString. * * The string can be later loaded with ValkeyModule_LoadString() or * other Load family functions expecting a serialized string inside * the RDB file. */ void VM_SaveString(ValkeyModuleIO *io, ValkeyModuleString *s) { if (io->error) return; if (flushValkeyModuleIOBuffer(io) == -1) goto saveerr; /* Save opcode. */ ssize_t retval = rdbSaveLen(io->rio, RDB_MODULE_OPCODE_STRING); if (retval == -1) goto saveerr; io->bytes += retval; /* Save value. */ retval = rdbSaveStringObject(io->rio, s); if (retval == -1) goto saveerr; io->bytes += retval; return; saveerr: io->error = 1; } /* Like ValkeyModule_SaveString() but takes a raw C pointer and length * as input. */ void VM_SaveStringBuffer(ValkeyModuleIO *io, const char *str, size_t len) { if (io->error) return; if (flushValkeyModuleIOBuffer(io) == -1) goto saveerr; /* Save opcode. */ ssize_t retval = rdbSaveLen(io->rio, RDB_MODULE_OPCODE_STRING); if (retval == -1) goto saveerr; io->bytes += retval; /* Save value. */ retval = rdbSaveRawString(io->rio, (unsigned char *)str, len); if (retval == -1) goto saveerr; io->bytes += retval; return; saveerr: io->error = 1; } /* Implements VM_LoadString() and VM_LoadStringBuffer() */ void *moduleLoadString(ValkeyModuleIO *io, int plain, size_t *lenptr) { if (io->error) return NULL; uint64_t opcode = rdbLoadLen(io->rio, NULL); if (opcode != RDB_MODULE_OPCODE_STRING) goto loaderr; void *s = rdbGenericLoadStringObject(io->rio, plain ? RDB_LOAD_PLAIN : RDB_LOAD_NONE, lenptr); if (s == NULL) goto loaderr; return s; loaderr: moduleRDBLoadError(io); return NULL; } /* In the context of the rdb_load method of a module data type, loads a string * from the RDB file, that was previously saved with ValkeyModule_SaveString() * functions family. * * The returned string is a newly allocated ValkeyModuleString object, and * the user should at some point free it with a call to ValkeyModule_FreeString(). * * If the data structure does not store strings as ValkeyModuleString objects, * the similar function ValkeyModule_LoadStringBuffer() could be used instead. */ ValkeyModuleString *VM_LoadString(ValkeyModuleIO *io) { return moduleLoadString(io, 0, NULL); } /* Like ValkeyModule_LoadString() but returns a heap allocated string that * was allocated with ValkeyModule_Alloc(), and can be resized or freed with * ValkeyModule_Realloc() or ValkeyModule_Free(). * * The size of the string is stored at '*lenptr' if not NULL. * The returned string is not automatically NULL terminated, it is loaded * exactly as it was stored inside the RDB file. */ char *VM_LoadStringBuffer(ValkeyModuleIO *io, size_t *lenptr) { return moduleLoadString(io, 1, lenptr); } /* In the context of the rdb_save method of a module data type, saves a double * value to the RDB file. The double can be a valid number, a NaN or infinity. * It is possible to load back the value with ValkeyModule_LoadDouble(). */ void VM_SaveDouble(ValkeyModuleIO *io, double value) { if (io->error) return; if (flushValkeyModuleIOBuffer(io) == -1) goto saveerr; /* Save opcode. */ int retval = rdbSaveLen(io->rio, RDB_MODULE_OPCODE_DOUBLE); if (retval == -1) goto saveerr; io->bytes += retval; /* Save value. */ retval = rdbSaveBinaryDoubleValue(io->rio, value); if (retval == -1) goto saveerr; io->bytes += retval; return; saveerr: io->error = 1; } /* In the context of the rdb_save method of a module data type, loads back the * double value saved by ValkeyModule_SaveDouble(). */ double VM_LoadDouble(ValkeyModuleIO *io) { if (io->error) return 0; uint64_t opcode = rdbLoadLen(io->rio, NULL); if (opcode != RDB_MODULE_OPCODE_DOUBLE) goto loaderr; double value; int retval = rdbLoadBinaryDoubleValue(io->rio, &value); if (retval == -1) goto loaderr; return value; loaderr: moduleRDBLoadError(io); return 0; } /* In the context of the rdb_save method of a module data type, saves a float * value to the RDB file. The float can be a valid number, a NaN or infinity. * It is possible to load back the value with ValkeyModule_LoadFloat(). */ void VM_SaveFloat(ValkeyModuleIO *io, float value) { if (io->error) return; if (flushValkeyModuleIOBuffer(io) == -1) goto saveerr; /* Save opcode. */ int retval = rdbSaveLen(io->rio, RDB_MODULE_OPCODE_FLOAT); if (retval == -1) goto saveerr; io->bytes += retval; /* Save value. */ retval = rdbSaveBinaryFloatValue(io->rio, value); if (retval == -1) goto saveerr; io->bytes += retval; return; saveerr: io->error = 1; } /* In the context of the rdb_save method of a module data type, loads back the * float value saved by ValkeyModule_SaveFloat(). */ float VM_LoadFloat(ValkeyModuleIO *io) { if (io->error) return 0; uint64_t opcode = rdbLoadLen(io->rio, NULL); if (opcode != RDB_MODULE_OPCODE_FLOAT) goto loaderr; float value; int retval = rdbLoadBinaryFloatValue(io->rio, &value); if (retval == -1) goto loaderr; return value; loaderr: moduleRDBLoadError(io); return 0; } /* In the context of the rdb_save method of a module data type, saves a long double * value to the RDB file. The double can be a valid number, a NaN or infinity. * It is possible to load back the value with ValkeyModule_LoadLongDouble(). */ void VM_SaveLongDouble(ValkeyModuleIO *io, long double value) { if (io->error) return; char buf[MAX_LONG_DOUBLE_CHARS]; /* Long double has different number of bits in different platforms, so we * save it as a string type. */ size_t len = ld2string(buf, sizeof(buf), value, LD_STR_HEX); VM_SaveStringBuffer(io, buf, len); } /* In the context of the rdb_save method of a module data type, loads back the * long double value saved by ValkeyModule_SaveLongDouble(). */ long double VM_LoadLongDouble(ValkeyModuleIO *io) { if (io->error) return 0; long double value; size_t len; char *str = VM_LoadStringBuffer(io, &len); if (!str) return 0; string2ld(str, len, &value); VM_Free(str); return value; } /* Iterate over modules, and trigger rdb aux saving for the ones modules types * who asked for it. */ ssize_t rdbSaveModulesAux(rio *rdb, int when) { size_t total_written = 0; dictIterator *di = dictGetIterator(modules); dictEntry *de; while ((de = dictNext(di)) != NULL) { struct ValkeyModule *module = dictGetVal(de); listIter li; listNode *ln; listRewind(module->types, &li); while ((ln = listNext(&li))) { moduleType *mt = ln->value; if ((!mt->aux_save && !mt->aux_save2) || !(mt->aux_save_triggers & when)) continue; ssize_t ret = rdbSaveSingleModuleAux(rdb, when, mt); if (ret == -1) { dictReleaseIterator(di); return -1; } total_written += ret; } } dictReleaseIterator(di); return total_written; } /* -------------------------------------------------------------------------- * ## Key digest API (DEBUG DIGEST interface for modules types) * -------------------------------------------------------------------------- */ /* Add a new element to the digest. This function can be called multiple times * one element after the other, for all the elements that constitute a given * data structure. The function call must be followed by the call to * `ValkeyModule_DigestEndSequence` eventually, when all the elements that are * always in a given order are added. See the Modules data types * documentation for more info. However this is a quick example that uses the * Set, Hash and List data types as an example. * * To add a sequence of unordered elements (for example in the case of a * Set), the pattern to use is: * * foreach element { * AddElement(element); * EndSequence(); * } * * Because Sets are not ordered, so every element added has a position that * does not depend from the other. However if instead our elements are * ordered in pairs, like field-value pairs of a Hash, then one should * use: * * foreach key,value { * AddElement(key); * AddElement(value); * EndSequence(); * } * * Because the key and value will be always in the above order, while instead * the single key-value pairs, can appear in any position into a hash. * * A list of ordered elements would be implemented with: * * foreach element { * AddElement(element); * } * EndSequence(); * */ void VM_DigestAddStringBuffer(ValkeyModuleDigest *md, const char *ele, size_t len) { mixDigest(md->o, ele, len); } /* Like `ValkeyModule_DigestAddStringBuffer()` but takes a `long long` as input * that gets converted into a string before adding it to the digest. */ void VM_DigestAddLongLong(ValkeyModuleDigest *md, long long ll) { char buf[LONG_STR_SIZE]; size_t len = ll2string(buf, sizeof(buf), ll); mixDigest(md->o, buf, len); } /* See the documentation for `ValkeyModule_DigestAddElement()`. */ void VM_DigestEndSequence(ValkeyModuleDigest *md) { xorDigest(md->x, md->o, sizeof(md->o)); memset(md->o, 0, sizeof(md->o)); } /* Decode a serialized representation of a module data type 'mt', in a specific encoding version 'encver' * from string 'str' and return a newly allocated value, or NULL if decoding failed. * * This call basically reuses the 'rdb_load' callback which module data types * implement in order to allow a module to arbitrarily serialize/de-serialize * keys, similar to how the 'DUMP' and 'RESTORE' commands are implemented. * * Modules should generally use the VALKEYMODULE_OPTIONS_HANDLE_IO_ERRORS flag and * make sure the de-serialization code properly checks and handles IO errors * (freeing allocated buffers and returning a NULL). * * If this is NOT done, the server will handle corrupted (or just truncated) serialized * data by producing an error message and terminating the process. */ void *VM_LoadDataTypeFromStringEncver(const ValkeyModuleString *str, const moduleType *mt, int encver) { rio payload; ValkeyModuleIO io; void *ret; rioInitWithBuffer(&payload, str->ptr); moduleInitIOContext(&io, (moduleType *)mt, &payload, NULL, -1); /* All VM_Save*() calls always write a version 2 compatible format, so we * need to make sure we read the same. */ ret = mt->rdb_load(&io, encver); if (io.ctx) { moduleFreeContext(io.ctx); zfree(io.ctx); } return ret; } /* Similar to VM_LoadDataTypeFromStringEncver, original version of the API, kept * for backward compatibility. */ void *VM_LoadDataTypeFromString(const ValkeyModuleString *str, const moduleType *mt) { return VM_LoadDataTypeFromStringEncver(str, mt, 0); } /* Encode a module data type 'mt' value 'data' into serialized form, and return it * as a newly allocated ValkeyModuleString. * * This call basically reuses the 'rdb_save' callback which module data types * implement in order to allow a module to arbitrarily serialize/de-serialize * keys, similar to how the 'DUMP' and 'RESTORE' commands are implemented. */ ValkeyModuleString *VM_SaveDataTypeToString(ValkeyModuleCtx *ctx, void *data, const moduleType *mt) { rio payload; ValkeyModuleIO io; rioInitWithBuffer(&payload, sdsempty()); moduleInitIOContext(&io, (moduleType *)mt, &payload, NULL, -1); mt->rdb_save(&io, data); if (io.ctx) { moduleFreeContext(io.ctx); zfree(io.ctx); } if (io.error) { return NULL; } else { robj *str = createObject(OBJ_STRING, payload.io.buffer.ptr); if (ctx != NULL) autoMemoryAdd(ctx, VALKEYMODULE_AM_STRING, str); return str; } } /* Returns the name of the key currently being processed. */ const ValkeyModuleString *VM_GetKeyNameFromDigest(ValkeyModuleDigest *dig) { return dig->key; } /* Returns the database id of the key currently being processed. */ int VM_GetDbIdFromDigest(ValkeyModuleDigest *dig) { return dig->dbid; } /* -------------------------------------------------------------------------- * ## AOF API for modules data types * -------------------------------------------------------------------------- */ /* Emits a command into the AOF during the AOF rewriting process. This function * is only called in the context of the aof_rewrite method of data types exported * by a module. The command works exactly like ValkeyModule_Call() in the way * the parameters are passed, but it does not return anything as the error * handling is performed by the server itself. */ void VM_EmitAOF(ValkeyModuleIO *io, const char *cmdname, const char *fmt, ...) { if (io->error) return; struct serverCommand *cmd; robj **argv = NULL; int argc = 0, flags = 0, j; va_list ap; cmd = lookupCommandByCString((char *)cmdname); if (!cmd) { serverLog(LL_WARNING, "Fatal: AOF method for module data type '%s' tried to " "emit unknown command '%s'", io->type->name, cmdname); io->error = 1; errno = EINVAL; return; } /* Emit the arguments into the AOF in RESP format. */ va_start(ap, fmt); argv = moduleCreateArgvFromUserFormat(cmdname, fmt, &argc, &flags, ap); va_end(ap); if (argv == NULL) { serverLog(LL_WARNING, "Fatal: AOF method for module data type '%s' tried to " "call ValkeyModule_EmitAOF() with wrong format specifiers '%s'", io->type->name, fmt); io->error = 1; errno = EINVAL; return; } /* Bulk count. */ if (!io->error && rioWriteBulkCount(io->rio, '*', argc) == 0) io->error = 1; /* Arguments. */ for (j = 0; j < argc; j++) { if (!io->error && rioWriteBulkObject(io->rio, argv[j]) == 0) io->error = 1; decrRefCount(argv[j]); } zfree(argv); return; } /* -------------------------------------------------------------------------- * ## IO context handling * -------------------------------------------------------------------------- */ ValkeyModuleCtx *VM_GetContextFromIO(ValkeyModuleIO *io) { if (io->ctx) return io->ctx; /* Can't have more than one... */ io->ctx = zmalloc(sizeof(ValkeyModuleCtx)); moduleCreateContext(io->ctx, io->type->module, VALKEYMODULE_CTX_NONE); return io->ctx; } /* Returns the name of the key currently being processed. * There is no guarantee that the key name is always available, so this may return NULL. */ const ValkeyModuleString *VM_GetKeyNameFromIO(ValkeyModuleIO *io) { return io->key; } /* Returns a ValkeyModuleString with the name of the key from ValkeyModuleKey. */ const ValkeyModuleString *VM_GetKeyNameFromModuleKey(ValkeyModuleKey *key) { return key ? key->key : NULL; } /* Returns a database id of the key from ValkeyModuleKey. */ int VM_GetDbIdFromModuleKey(ValkeyModuleKey *key) { return key ? key->db->id : -1; } /* Returns the database id of the key currently being processed. * There is no guarantee that this info is always available, so this may return -1. */ int VM_GetDbIdFromIO(ValkeyModuleIO *io) { return io->dbid; } /* -------------------------------------------------------------------------- * ## Logging * -------------------------------------------------------------------------- */ /* This is the low level function implementing both: * * VM_Log() * VM_LogIOError() * */ void moduleLogRaw(ValkeyModule *module, const char *levelstr, const char *fmt, va_list ap) { char msg[LOG_MAX_LEN]; size_t name_len; int level; if (!strcasecmp(levelstr, "debug")) level = LL_DEBUG; else if (!strcasecmp(levelstr, "verbose")) level = LL_VERBOSE; else if (!strcasecmp(levelstr, "notice")) level = LL_NOTICE; else if (!strcasecmp(levelstr, "warning")) level = LL_WARNING; else level = LL_VERBOSE; /* Default. */ if (level < server.verbosity) return; name_len = snprintf(msg, sizeof(msg), "<%s> ", module ? module->name : "module"); vsnprintf(msg + name_len, sizeof(msg) - name_len, fmt, ap); serverLogRaw(level, msg); } /* Produces a log message to the standard server log, the format accepts * printf-alike specifiers, while level is a string describing the log * level to use when emitting the log, and must be one of the following: * * * "debug" (`VALKEYMODULE_LOGLEVEL_DEBUG`) * * "verbose" (`VALKEYMODULE_LOGLEVEL_VERBOSE`) * * "notice" (`VALKEYMODULE_LOGLEVEL_NOTICE`) * * "warning" (`VALKEYMODULE_LOGLEVEL_WARNING`) * * If the specified log level is invalid, verbose is used by default. * There is a fixed limit to the length of the log line this function is able * to emit, this limit is not specified but is guaranteed to be more than * a few lines of text. * * The ctx argument may be NULL if cannot be provided in the context of the * caller for instance threads or callbacks, in which case a generic "module" * will be used instead of the module name. */ void VM_Log(ValkeyModuleCtx *ctx, const char *levelstr, const char *fmt, ...) { va_list ap; va_start(ap, fmt); moduleLogRaw(ctx ? ctx->module : NULL, levelstr, fmt, ap); va_end(ap); } /* Log errors from RDB / AOF serialization callbacks. * * This function should be used when a callback is returning a critical * error to the caller since cannot load or save the data for some * critical reason. */ void VM_LogIOError(ValkeyModuleIO *io, const char *levelstr, const char *fmt, ...) { va_list ap; va_start(ap, fmt); moduleLogRaw(io->type->module, levelstr, fmt, ap); va_end(ap); } /* Valkey assert function. * * The macro `ValkeyModule_Assert(expression)` is recommended, rather than * calling this function directly. * * A failed assertion will shut down the server and produce logging information * that looks identical to information generated by the server itself. */ void VM__Assert(const char *estr, const char *file, int line) { _serverAssert(estr, file, line); } /* Allows adding event to the latency monitor to be observed by the LATENCY * command. The call is skipped if the latency is smaller than the configured * latency-monitor-threshold. */ void VM_LatencyAddSample(const char *event, mstime_t latency) { latencyAddSampleIfNeeded(event, latency); } /* -------------------------------------------------------------------------- * ## Blocking clients from modules * * For a guide about blocking commands in modules, see * https://valkey.io/topics/modules-blocking-ops. * -------------------------------------------------------------------------- */ /* Returns 1 if the client already in the moduleUnblocked list, 0 otherwise. */ int isModuleClientUnblocked(client *c) { ValkeyModuleBlockedClient *bc = c->bstate->module_blocked_handle; return bc->unblocked == 1; } /* This is called from blocked.c in order to unblock a client: may be called * for multiple reasons while the client is in the middle of being blocked * because the client is terminated, but is also called for cleanup when a * client is unblocked in a clean way after replaying. * * What we do here is just to set the client to NULL in the module * blocked client handle. This way if the client is terminated while there * is a pending threaded operation involving the blocked client, we'll know * that the client no longer exists and no reply callback should be called. * * The structure ValkeyModuleBlockedClient will be always deallocated when * running the list of clients blocked by a module that need to be unblocked. */ void unblockClientFromModule(client *c) { ValkeyModuleBlockedClient *bc = c->bstate->module_blocked_handle; /* Call the disconnection callback if any. Note that * bc->disconnect_callback is set to NULL if the client gets disconnected * by the module itself or because of a timeout, so the callback will NOT * get called if this is not an actual disconnection event. */ if (bc->disconnect_callback) { ValkeyModuleCtx ctx; moduleCreateContext(&ctx, bc->module, VALKEYMODULE_CTX_NONE); ctx.blocked_privdata = bc->privdata; ctx.client = bc->client; bc->disconnect_callback(&ctx, bc); moduleFreeContext(&ctx); } /* If we made it here and client is still blocked it means that the command * timed-out, client was killed or disconnected and disconnect_callback was * not implemented (or it was, but VM_UnblockClient was not called from * within it, as it should). * We must call moduleUnblockClient in order to free privdata and * ValkeyModuleBlockedClient. * * Note that we only do that for clients that are blocked on keys, for which * the contract is that the module should not call VM_UnblockClient under * normal circumstances. * Clients implementing threads and working with private data should be * aware that calling VM_UnblockClient for every blocked client is their * responsibility, and if they fail to do so memory may leak. Ideally they * should implement the disconnect and timeout callbacks and call * VM_UnblockClient, but any other way is also acceptable. */ if (bc->blocked_on_keys && !bc->unblocked) moduleUnblockClient(c); bc->client = NULL; } /* Block a client in the context of a module: this function implements both * VM_BlockClient() and VM_BlockClientOnKeys() depending on the fact the * keys are passed or not. * * When not blocking for keys, the keys, numkeys, and privdata parameters are * not needed. The privdata in that case must be NULL, since later is * VM_UnblockClient() that will provide some private data that the reply * callback will receive. * * Instead when blocking for keys, normally VM_UnblockClient() will not be * called (because the client will unblock when the key is modified), so * 'privdata' should be provided in that case, so that once the client is * unlocked and the reply callback is called, it will receive its associated * private data. * * Even when blocking on keys, VM_UnblockClient() can be called however, but * in that case the privdata argument is disregarded, because we pass the * reply callback the privdata that is set here while blocking. * */ ValkeyModuleBlockedClient *moduleBlockClient(ValkeyModuleCtx *ctx, ValkeyModuleCmdFunc reply_callback, ValkeyModuleAuthCallback auth_reply_callback, ValkeyModuleCmdFunc timeout_callback, void (*free_privdata)(ValkeyModuleCtx *, void *), long long timeout_ms, ValkeyModuleString **keys, int numkeys, void *privdata, int flags) { client *c = ctx->client; int islua = scriptIsRunning(); int ismulti = server.in_exec; initClientBlockingState(c); c->bstate->module_blocked_handle = zmalloc(sizeof(ValkeyModuleBlockedClient)); ValkeyModuleBlockedClient *bc = c->bstate->module_blocked_handle; ctx->module->blocked_clients++; /* We need to handle the invalid operation of calling modules blocking * commands from Lua or MULTI. We actually create an already aborted * (client set to NULL) blocked client handle, and actually reply with * an error. */ bc->client = (islua || ismulti) ? NULL : c; bc->module = ctx->module; bc->reply_callback = reply_callback; bc->auth_reply_cb = auth_reply_callback; bc->timeout_callback = timeout_callback; bc->disconnect_callback = NULL; /* Set by VM_SetDisconnectCallback() */ bc->free_privdata = free_privdata; bc->privdata = privdata; bc->reply_client = moduleAllocTempClient(); bc->thread_safe_ctx_client = moduleAllocTempClient(); if (bc->client) bc->reply_client->resp = bc->client->resp; bc->dbid = c->db->id; bc->blocked_on_keys = keys != NULL; bc->unblocked = 0; bc->background_timer = 0; bc->background_duration = 0; mstime_t timeout = 0; if (timeout_ms) { mstime_t now = mstime(); if (timeout_ms > LLONG_MAX - now) { c->bstate->module_blocked_handle = NULL; addReplyError(c, "timeout is out of range"); /* 'timeout_ms+now' would overflow */ return bc; } timeout = timeout_ms + now; } if (islua || ismulti) { c->bstate->module_blocked_handle = NULL; addReplyError(c, islua ? "Blocking module command called from Lua script" : "Blocking module command called from transaction"); } else if (ctx->flags & VALKEYMODULE_CTX_BLOCKED_REPLY) { c->bstate->module_blocked_handle = NULL; addReplyError(c, "Blocking module command called from a Reply callback context"); } else if (!auth_reply_callback && clientHasModuleAuthInProgress(c)) { c->bstate->module_blocked_handle = NULL; addReplyError(c, "Clients undergoing module based authentication can only be blocked on auth"); } else { if (keys) { blockForKeys(c, BLOCKED_MODULE, keys, numkeys, timeout, flags & VALKEYMODULE_BLOCK_UNBLOCK_DELETED); } else { c->bstate->timeout = timeout; blockClient(c, BLOCKED_MODULE); } } return bc; } /* This API registers a callback to execute in addition to normal password based authentication. * Multiple callbacks can be registered across different modules. When a Module is unloaded, all the * auth callbacks registered by it are unregistered. * The callbacks are attempted (in the order of most recently registered first) when the AUTH/HELLO * (with AUTH field provided) commands are called. * The callbacks will be called with a module context along with a username and a password, and are * expected to take one of the following actions: * (1) Authenticate - Use the VM_AuthenticateClient* API and return VALKEYMODULE_AUTH_HANDLED. * This will immediately end the auth chain as successful and add the OK reply. * (2) Deny Authentication - Return VALKEYMODULE_AUTH_HANDLED without authenticating or blocking the * client. Optionally, `err` can be set to a custom error message and `err` will be automatically * freed by the server. * This will immediately end the auth chain as unsuccessful and add the ERR reply. * (3) Block a client on authentication - Use the VM_BlockClientOnAuth API and return * VALKEYMODULE_AUTH_HANDLED. Here, the client will be blocked until the VM_UnblockClient API is used * which will trigger the auth reply callback (provided through the VM_BlockClientOnAuth). * In this reply callback, the Module should authenticate, deny or skip handling authentication. * (4) Skip handling Authentication - Return VALKEYMODULE_AUTH_NOT_HANDLED without blocking the * client. This will allow the engine to attempt the next module auth callback. * If none of the callbacks authenticate or deny auth, then password based auth is attempted and * will authenticate or add failure logs and reply to the clients accordingly. * * Note: If a client is disconnected while it was in the middle of blocking module auth, that * occurrence of the AUTH or HELLO command will not be tracked in the INFO command stats. * * The following is an example of how non-blocking module based authentication can be used: * * int auth_cb(ValkeyModuleCtx *ctx, ValkeyModuleString *username, ValkeyModuleString *password, ValkeyModuleString * **err) { const char *user = ValkeyModule_StringPtrLen(username, NULL); const char *pwd = * ValkeyModule_StringPtrLen(password, NULL); if (!strcmp(user,"foo") && !strcmp(pwd,"valid_password")) { * ValkeyModule_AuthenticateClientWithACLUser(ctx, "foo", 3, NULL, NULL, NULL); * return VALKEYMODULE_AUTH_HANDLED; * } * * else if (!strcmp(user,"foo") && !strcmp(pwd,"wrong_password")) { * ValkeyModuleString *log = ValkeyModule_CreateString(ctx, "Module Auth", 11); * ValkeyModule_ACLAddLogEntryByUserName(ctx, username, log, VALKEYMODULE_ACL_LOG_AUTH); * ValkeyModule_FreeString(ctx, log); * const char *err_msg = "Auth denied by Misc Module."; * *err = ValkeyModule_CreateString(ctx, err_msg, strlen(err_msg)); * return VALKEYMODULE_AUTH_HANDLED; * } * return VALKEYMODULE_AUTH_NOT_HANDLED; * } * * int ValkeyModule_OnLoad(ValkeyModuleCtx *ctx, ValkeyModuleString **argv, int argc) { * if (ValkeyModule_Init(ctx,"authmodule",1,VALKEYMODULE_APIVER_1)== VALKEYMODULE_ERR) * return VALKEYMODULE_ERR; * ValkeyModule_RegisterAuthCallback(ctx, auth_cb); * return VALKEYMODULE_OK; * } */ void VM_RegisterAuthCallback(ValkeyModuleCtx *ctx, ValkeyModuleAuthCallback cb) { ValkeyModuleAuthCtx *auth_ctx = zmalloc(sizeof(ValkeyModuleAuthCtx)); auth_ctx->module = ctx->module; auth_ctx->auth_cb = cb; listAddNodeHead(moduleAuthCallbacks, auth_ctx); } /* Helper function to invoke the free private data callback of a Module blocked client. */ void moduleInvokeFreePrivDataCallback(client *c, ValkeyModuleBlockedClient *bc) { if (bc->privdata && bc->free_privdata) { ValkeyModuleCtx ctx; int ctx_flags = c == NULL ? VALKEYMODULE_CTX_BLOCKED_DISCONNECTED : VALKEYMODULE_CTX_NONE; moduleCreateContext(&ctx, bc->module, ctx_flags); ctx.blocked_privdata = bc->privdata; ctx.client = bc->client; bc->free_privdata(&ctx, bc->privdata); moduleFreeContext(&ctx); } } /* Unregisters all the module auth callbacks that have been registered by this Module. */ void moduleUnregisterAuthCBs(ValkeyModule *module) { listIter li; listNode *ln; listRewind(moduleAuthCallbacks, &li); while ((ln = listNext(&li))) { ValkeyModuleAuthCtx *ctx = listNodeValue(ln); if (ctx->module == module) { listDelNode(moduleAuthCallbacks, ln); zfree(ctx); } } } /* Search for & attempt next module auth callback after skipping the ones already attempted. * Returns the result of the module auth callback. */ int attemptNextAuthCb(client *c, robj *username, robj *password, robj **err) { int handle_next_callback = (!c->module_data || c->module_data->module_auth_ctx == NULL); ValkeyModuleAuthCtx *cur_auth_ctx = NULL; listNode *ln; listIter li; listRewind(moduleAuthCallbacks, &li); int result = VALKEYMODULE_AUTH_NOT_HANDLED; while ((ln = listNext(&li))) { cur_auth_ctx = listNodeValue(ln); /* Skip over the previously attempted auth contexts. */ if (!handle_next_callback) { handle_next_callback = cur_auth_ctx == c->module_data->module_auth_ctx; continue; } /* Remove the module auth complete flag before we attempt the next cb. */ c->flag.module_auth_has_result = 0; ValkeyModuleCtx ctx; moduleCreateContext(&ctx, cur_auth_ctx->module, VALKEYMODULE_CTX_NONE); ctx.client = c; *err = NULL; initClientModuleData(c); c->module_data->module_auth_ctx = cur_auth_ctx; result = cur_auth_ctx->auth_cb(&ctx, username, password, err); moduleFreeContext(&ctx); if (result == VALKEYMODULE_AUTH_HANDLED) break; /* If Auth was not handled (allowed/denied/blocked) by the Module, try the next auth cb. */ } return result; } /* Helper function to handle a reprocessed unblocked auth client. * Returns VALKEYMODULE_AUTH_NOT_HANDLED if the client was not reprocessed after a blocking module * auth operation. * Otherwise, we attempt the auth reply callback & the free priv data callback, update fields and * return the result of the reply callback. */ int attemptBlockedAuthReplyCallback(client *c, robj *username, robj *password, robj **err) { int result = VALKEYMODULE_AUTH_NOT_HANDLED; if (!c->module_data || !c->module_data->module_blocked_client) return result; ValkeyModuleBlockedClient *bc = (ValkeyModuleBlockedClient *)c->module_data->module_blocked_client; bc->client = c; if (bc->auth_reply_cb) { ValkeyModuleCtx ctx; moduleCreateContext(&ctx, bc->module, VALKEYMODULE_CTX_BLOCKED_REPLY); ctx.blocked_privdata = bc->privdata; ctx.blocked_ready_key = NULL; ctx.client = bc->client; ctx.blocked_client = bc; result = bc->auth_reply_cb(&ctx, username, password, err); moduleFreeContext(&ctx); } moduleInvokeFreePrivDataCallback(c, bc); c->module_data->module_blocked_client = NULL; c->lastcmd->microseconds += bc->background_duration; bc->module->blocked_clients--; zfree(bc); return result; } /* Helper function to attempt Module based authentication through module auth callbacks. * Here, the Module is expected to authenticate the client using the ValkeyModule APIs and to add ACL * logs in case of errors. * Returns one of the following codes: * AUTH_OK - Indicates that a module handled and authenticated the client. * AUTH_ERR - Indicates that a module handled and denied authentication for this client. * AUTH_NOT_HANDLED - Indicates that authentication was not handled by any Module and that * normal password based authentication can be attempted next. * AUTH_BLOCKED - Indicates module authentication is in progress through a blocking implementation. * In this case, authentication is handled here again after the client is unblocked / reprocessed. */ int checkModuleAuthentication(client *c, robj *username, robj *password, robj **err) { if (!listLength(moduleAuthCallbacks)) return AUTH_NOT_HANDLED; int result = attemptBlockedAuthReplyCallback(c, username, password, err); if (result == VALKEYMODULE_AUTH_NOT_HANDLED) { result = attemptNextAuthCb(c, username, password, err); } if (c->flag.blocked) { /* Modules are expected to return VALKEYMODULE_AUTH_HANDLED when blocking clients. */ serverAssert(result == VALKEYMODULE_AUTH_HANDLED); return AUTH_BLOCKED; } if (c->module_data) c->module_data->module_auth_ctx = NULL; if (result == VALKEYMODULE_AUTH_NOT_HANDLED) { c->flag.module_auth_has_result = 0; return AUTH_NOT_HANDLED; } if (c->flag.module_auth_has_result) { c->flag.module_auth_has_result = 0; if (c->flag.authenticated) return AUTH_OK; } return AUTH_ERR; } /* This function is called from module.c in order to check if a module * blocked for BLOCKED_MODULE and subtype 'on keys' (bc->blocked_on_keys true) * can really be unblocked, since the module was able to serve the client. * If the callback returns VALKEYMODULE_OK, then the client can be unblocked, * otherwise the client remains blocked and we'll retry again when one of * the keys it blocked for becomes "ready" again. * This function returns 1 if client was served (and should be unblocked) */ int moduleTryServeClientBlockedOnKey(client *c, robj *key) { int served = 0; ValkeyModuleBlockedClient *bc = c->bstate->module_blocked_handle; /* Protect against re-processing: don't serve clients that are already * in the unblocking list for any reason (including VM_UnblockClient() * explicit call). See #6798. */ if (bc->unblocked) return 0; ValkeyModuleCtx ctx; moduleCreateContext(&ctx, bc->module, VALKEYMODULE_CTX_BLOCKED_REPLY); ctx.blocked_ready_key = key; ctx.blocked_privdata = bc->privdata; ctx.client = bc->client; ctx.blocked_client = bc; if (bc->reply_callback(&ctx, (void **)c->argv, c->argc) == VALKEYMODULE_OK) served = 1; moduleFreeContext(&ctx); return served; } /* Block a client in the context of a blocking command, returning a handle * which will be used, later, in order to unblock the client with a call to * ValkeyModule_UnblockClient(). The arguments specify callback functions * and a timeout after which the client is unblocked. * * The callbacks are called in the following contexts: * * reply_callback: called after a successful ValkeyModule_UnblockClient() * call in order to reply to the client and unblock it. * * timeout_callback: called when the timeout is reached or if `CLIENT UNBLOCK` * is invoked, in order to send an error to the client. * * free_privdata: called in order to free the private data that is passed * by ValkeyModule_UnblockClient() call. * * Note: ValkeyModule_UnblockClient should be called for every blocked client, * even if client was killed, timed-out or disconnected. Failing to do so * will result in memory leaks. * * There are some cases where ValkeyModule_BlockClient() cannot be used: * * 1. If the client is a Lua script. * 2. If the client is executing a MULTI block. * * In these cases, a call to ValkeyModule_BlockClient() will **not** block the * client, but instead produce a specific error reply. * * A module that registers a timeout_callback function can also be unblocked * using the `CLIENT UNBLOCK` command, which will trigger the timeout callback. * If a callback function is not registered, then the blocked client will be * treated as if it is not in a blocked state and `CLIENT UNBLOCK` will return * a zero value. * * Measuring background time: By default the time spent in the blocked command * is not account for the total command duration. To include such time you should * use VM_BlockedClientMeasureTimeStart() and VM_BlockedClientMeasureTimeEnd() one, * or multiple times within the blocking command background work. */ ValkeyModuleBlockedClient *VM_BlockClient(ValkeyModuleCtx *ctx, ValkeyModuleCmdFunc reply_callback, ValkeyModuleCmdFunc timeout_callback, void (*free_privdata)(ValkeyModuleCtx *, void *), long long timeout_ms) { return moduleBlockClient(ctx, reply_callback, NULL, timeout_callback, free_privdata, timeout_ms, NULL, 0, NULL, 0); } /* Block the current client for module authentication in the background. If module auth is not in * progress on the client, the API returns NULL. Otherwise, the client is blocked and the VM_BlockedClient * is returned similar to the VM_BlockClient API. * Note: Only use this API from the context of a module auth callback. */ ValkeyModuleBlockedClient *VM_BlockClientOnAuth(ValkeyModuleCtx *ctx, ValkeyModuleAuthCallback reply_callback, void (*free_privdata)(ValkeyModuleCtx *, void *)) { if (!clientHasModuleAuthInProgress(ctx->client)) { addReplyError(ctx->client, "Module blocking client on auth when not currently undergoing module authentication"); return NULL; } ValkeyModuleBlockedClient *bc = moduleBlockClient(ctx, NULL, reply_callback, NULL, free_privdata, 0, NULL, 0, NULL, 0); if (ctx->client->flag.blocked) { ctx->client->flag.pending_command = 1; } return bc; } /* Get the private data that was previusely set on a blocked client */ void *VM_BlockClientGetPrivateData(ValkeyModuleBlockedClient *blocked_client) { return blocked_client->privdata; } /* Set private data on a blocked client */ void VM_BlockClientSetPrivateData(ValkeyModuleBlockedClient *blocked_client, void *private_data) { blocked_client->privdata = private_data; } /* This call is similar to ValkeyModule_BlockClient(), however in this case we * don't just block the client, but also ask the server to unblock it automatically * once certain keys become "ready", that is, contain more data. * * Basically this is similar to what a typical command usually does, * like BLPOP or BZPOPMAX: the client blocks if it cannot be served ASAP, * and later when the key receives new data (a list push for instance), the * client is unblocked and served. * * However in the case of this module API, when the client is unblocked? * * 1. If you block on a key of a type that has blocking operations associated, * like a list, a sorted set, a stream, and so forth, the client may be * unblocked once the relevant key is targeted by an operation that normally * unblocks the native blocking operations for that type. So if we block * on a list key, an RPUSH command may unblock our client and so forth. * 2. If you are implementing your native data type, or if you want to add new * unblocking conditions in addition to "1", you can call the modules API * ValkeyModule_SignalKeyAsReady(). * * Anyway we can't be sure if the client should be unblocked just because the * key is signaled as ready: for instance a successive operation may change the * key, or a client in queue before this one can be served, modifying the key * as well and making it empty again. So when a client is blocked with * ValkeyModule_BlockClientOnKeys() the reply callback is not called after * VM_UnblockClient() is called, but every time a key is signaled as ready: * if the reply callback can serve the client, it returns VALKEYMODULE_OK * and the client is unblocked, otherwise it will return VALKEYMODULE_ERR * and we'll try again later. * * The reply callback can access the key that was signaled as ready by * calling the API ValkeyModule_GetBlockedClientReadyKey(), that returns * just the string name of the key as a ValkeyModuleString object. * * Thanks to this system we can setup complex blocking scenarios, like * unblocking a client only if a list contains at least 5 items or other * more fancy logics. * * Note that another difference with ValkeyModule_BlockClient(), is that here * we pass the private data directly when blocking the client: it will * be accessible later in the reply callback. Normally when blocking with * ValkeyModule_BlockClient() the private data to reply to the client is * passed when calling ValkeyModule_UnblockClient() but here the unblocking * is performed by the server itself, so we need to have some private data before * hand. The private data is used to store any information about the specific * unblocking operation that you are implementing. Such information will be * freed using the free_privdata callback provided by the user. * * However the reply callback will be able to access the argument vector of * the command, so the private data is often not needed. * * Note: Under normal circumstances ValkeyModule_UnblockClient should not be * called for clients that are blocked on keys (Either the key will * become ready or a timeout will occur). If for some reason you do want * to call ValkeyModule_UnblockClient it is possible: Client will be * handled as if it were timed-out (You must implement the timeout * callback in that case). */ ValkeyModuleBlockedClient *VM_BlockClientOnKeys(ValkeyModuleCtx *ctx, ValkeyModuleCmdFunc reply_callback, ValkeyModuleCmdFunc timeout_callback, void (*free_privdata)(ValkeyModuleCtx *, void *), long long timeout_ms, ValkeyModuleString **keys, int numkeys, void *privdata) { return moduleBlockClient(ctx, reply_callback, NULL, timeout_callback, free_privdata, timeout_ms, keys, numkeys, privdata, 0); } /* Same as ValkeyModule_BlockClientOnKeys, but can take VALKEYMODULE_BLOCK_* flags * Can be either VALKEYMODULE_BLOCK_UNBLOCK_DEFAULT, which means default behavior (same * as calling ValkeyModule_BlockClientOnKeys) * * The flags is a bit mask of these: * * - `VALKEYMODULE_BLOCK_UNBLOCK_DELETED`: The clients should to be awakened in case any of `keys` are deleted. * Mostly useful for commands that require the key to exist (like XREADGROUP) */ ValkeyModuleBlockedClient *VM_BlockClientOnKeysWithFlags(ValkeyModuleCtx *ctx, ValkeyModuleCmdFunc reply_callback, ValkeyModuleCmdFunc timeout_callback, void (*free_privdata)(ValkeyModuleCtx *, void *), long long timeout_ms, ValkeyModuleString **keys, int numkeys, void *privdata, int flags) { return moduleBlockClient(ctx, reply_callback, NULL, timeout_callback, free_privdata, timeout_ms, keys, numkeys, privdata, flags); } /* This function is used in order to potentially unblock a client blocked * on keys with ValkeyModule_BlockClientOnKeys(). When this function is called, * all the clients blocked for this key will get their reply_callback called. */ void VM_SignalKeyAsReady(ValkeyModuleCtx *ctx, ValkeyModuleString *key) { signalKeyAsReady(ctx->client->db, key, OBJ_MODULE); } /* Implements VM_UnblockClient() and moduleUnblockClient(). */ int moduleUnblockClientByHandle(ValkeyModuleBlockedClient *bc, void *privdata) { pthread_mutex_lock(&moduleUnblockedClientsMutex); if (!bc->blocked_on_keys) bc->privdata = privdata; bc->unblocked = 1; if (listLength(moduleUnblockedClients) == 0) { if (write(server.module_pipe[1], "A", 1) != 1) { /* Ignore the error, this is best-effort. */ } } listAddNodeTail(moduleUnblockedClients, bc); pthread_mutex_unlock(&moduleUnblockedClientsMutex); return VALKEYMODULE_OK; } /* This API is used by the server core to unblock a client that was blocked * by a module. */ void moduleUnblockClient(client *c) { ValkeyModuleBlockedClient *bc = c->bstate->module_blocked_handle; moduleUnblockClientByHandle(bc, NULL); } /* Return true if the client 'c' was blocked by a module using * VM_BlockClientOnKeys(). */ int moduleClientIsBlockedOnKeys(client *c) { ValkeyModuleBlockedClient *bc = c->bstate->module_blocked_handle; return bc->blocked_on_keys; } /* Unblock a client blocked by `ValkeyModule_BlockedClient`. This will trigger * the reply callbacks to be called in order to reply to the client. * The 'privdata' argument will be accessible by the reply callback, so * the caller of this function can pass any value that is needed in order to * actually reply to the client. * * A common usage for 'privdata' is a thread that computes something that * needs to be passed to the client, included but not limited some slow * to compute reply or some reply obtained via networking. * * Note 1: this function can be called from threads spawned by the module. * * Note 2: when we unblock a client that is blocked for keys using the API * ValkeyModule_BlockClientOnKeys(), the privdata argument here is not used. * Unblocking a client that was blocked for keys using this API will still * require the client to get some reply, so the function will use the * "timeout" handler in order to do so (The privdata provided in * ValkeyModule_BlockClientOnKeys() is accessible from the timeout * callback via VM_GetBlockedClientPrivateData). */ int VM_UnblockClient(ValkeyModuleBlockedClient *bc, void *privdata) { if (bc->blocked_on_keys) { /* In theory the user should always pass the timeout handler as an * argument, but better to be safe than sorry. */ if (bc->timeout_callback == NULL) return VALKEYMODULE_ERR; if (bc->unblocked) return VALKEYMODULE_OK; if (bc->client) moduleBlockedClientTimedOut(bc->client, 1); } moduleUnblockClientByHandle(bc, privdata); return VALKEYMODULE_OK; } /* Abort a blocked client blocking operation: the client will be unblocked * without firing any callback. */ int VM_AbortBlock(ValkeyModuleBlockedClient *bc) { bc->reply_callback = NULL; bc->disconnect_callback = NULL; bc->auth_reply_cb = NULL; return VM_UnblockClient(bc, NULL); } /* Set a callback that will be called if a blocked client disconnects * before the module has a chance to call ValkeyModule_UnblockClient() * * Usually what you want to do there, is to cleanup your module state * so that you can call ValkeyModule_UnblockClient() safely, otherwise * the client will remain blocked forever if the timeout is large. * * Notes: * * 1. It is not safe to call Reply* family functions here, it is also * useless since the client is gone. * * 2. This callback is not called if the client disconnects because of * a timeout. In such a case, the client is unblocked automatically * and the timeout callback is called. */ void VM_SetDisconnectCallback(ValkeyModuleBlockedClient *bc, ValkeyModuleDisconnectFunc callback) { bc->disconnect_callback = callback; } /* This function will check the moduleUnblockedClients queue in order to * call the reply callback and really unblock the client. * * Clients end into this list because of calls to VM_UnblockClient(), * however it is possible that while the module was doing work for the * blocked client, it was terminated by the server (for timeout or other reasons). * When this happens the ValkeyModuleBlockedClient structure in the queue * will have the 'client' field set to NULL. */ void moduleHandleBlockedClients(void) { listNode *ln; ValkeyModuleBlockedClient *bc; pthread_mutex_lock(&moduleUnblockedClientsMutex); while (listLength(moduleUnblockedClients)) { ln = listFirst(moduleUnblockedClients); bc = ln->value; client *c = bc->client; listDelNode(moduleUnblockedClients, ln); pthread_mutex_unlock(&moduleUnblockedClientsMutex); /* Release the lock during the loop, as long as we don't * touch the shared list. */ /* Call the reply callback if the client is valid and we have * any callback. However the callback is not called if the client * was blocked on keys (VM_BlockClientOnKeys()), because we already * called such callback in moduleTryServeClientBlockedOnKey() when * the key was signaled as ready. */ long long prev_error_replies = server.stat_total_error_replies; uint64_t reply_us = 0; if (c && !bc->blocked_on_keys && bc->reply_callback) { ValkeyModuleCtx ctx; moduleCreateContext(&ctx, bc->module, VALKEYMODULE_CTX_BLOCKED_REPLY); ctx.blocked_privdata = bc->privdata; ctx.blocked_ready_key = NULL; ctx.client = bc->client; ctx.blocked_client = bc; monotime replyTimer; elapsedStart(&replyTimer); bc->reply_callback(&ctx, (void **)c->argv, c->argc); reply_us = elapsedUs(replyTimer); moduleFreeContext(&ctx); } /* Hold onto the blocked client if module auth is in progress. The reply callback is invoked * when the client is reprocessed. */ if (c && clientHasModuleAuthInProgress(c)) { c->module_data->module_blocked_client = bc; } else { /* Free privdata if any. */ moduleInvokeFreePrivDataCallback(c, bc); } /* It is possible that this blocked client object accumulated * replies to send to the client in a thread safe context. * We need to glue such replies to the client output buffer and * free the temporary client we just used for the replies. */ if (c) AddReplyFromClient(c, bc->reply_client); moduleReleaseTempClient(bc->reply_client); moduleReleaseTempClient(bc->thread_safe_ctx_client); /* Update stats now that we've finished the blocking operation. * This needs to be out of the reply callback above given that a * module might not define any callback and still do blocking ops. */ if (c && !clientHasModuleAuthInProgress(c)) { int had_errors = c->deferred_reply_errors ? !!listLength(c->deferred_reply_errors) : (server.stat_total_error_replies != prev_error_replies); updateStatsOnUnblock(c, bc->background_duration, reply_us, (had_errors ? ERROR_COMMAND_FAILED : 0)); } if (c != NULL) { /* Before unblocking the client, set the disconnect callback * to NULL, because if we reached this point, the client was * properly unblocked by the module. */ bc->disconnect_callback = NULL; unblockClient(c, 1); /* Update the wait offset, we don't know if this blocked client propagated anything, * currently we rather not add any API for that, so we just assume it did. */ c->woff = server.primary_repl_offset; /* Put the client in the list of clients that need to write * if there are pending replies here. This is needed since * during a non blocking command the client may receive output. */ if (!clientHasModuleAuthInProgress(c) && clientHasPendingReplies(c) && !c->flag.pending_write && c->conn) { c->flag.pending_write = 1; listLinkNodeHead(server.clients_pending_write, &c->clients_pending_write_node); } } /* Free 'bc' only after unblocking the client, since it is * referenced in the client blocking context, and must be valid * when calling unblockClient(). */ if (!(c && clientHasModuleAuthInProgress(c))) { bc->module->blocked_clients--; zfree(bc); } /* Lock again before to iterate the loop. */ pthread_mutex_lock(&moduleUnblockedClientsMutex); } pthread_mutex_unlock(&moduleUnblockedClientsMutex); } /* Check if the specified client can be safely timed out using * moduleBlockedClientTimedOut(). */ int moduleBlockedClientMayTimeout(client *c) { if (c->bstate->btype != BLOCKED_MODULE) return 1; ValkeyModuleBlockedClient *bc = c->bstate->module_blocked_handle; return (bc && bc->timeout_callback != NULL); } /* Called when our client timed out. After this function unblockClient() * is called, and it will invalidate the blocked client. So this function * does not need to do any cleanup. Eventually the module will call the * API to unblock the client and the memory will be released. * * If this function is called from a module, we handle the timeout callback * and the update of the unblock status in a thread-safe manner to avoid race * conditions with the main thread. * If this function is called from the main thread, we must handle the unblocking * of the client synchronously. This ensures that we can reply to the client before * resetClient() is called. */ void moduleBlockedClientTimedOut(client *c, int from_module) { ValkeyModuleBlockedClient *bc = c->bstate->module_blocked_handle; /* Protect against re-processing: don't serve clients that are already * in the unblocking list for any reason (including VM_UnblockClient() * explicit call). See #6798. */ if (bc->unblocked) return; ValkeyModuleCtx ctx; int flags = VALKEYMODULE_CTX_BLOCKED_TIMEOUT; if (from_module) flags |= VALKEYMODULE_CTX_THREAD_SAFE; moduleCreateContext(&ctx, bc->module, flags); ctx.client = bc->client; ctx.blocked_client = bc; ctx.blocked_privdata = bc->privdata; long long prev_error_replies; if (!from_module) prev_error_replies = server.stat_total_error_replies; if (bc->timeout_callback) { /* In theory, the user should always pass the timeout handler as an * argument, but better to be safe than sorry. */ bc->timeout_callback(&ctx, (void **)c->argv, c->argc); } moduleFreeContext(&ctx); if (!from_module) updateStatsOnUnblock(c, bc->background_duration, 0, ((server.stat_total_error_replies != prev_error_replies) ? ERROR_COMMAND_FAILED : 0)); /* For timeout events, we do not want to call the disconnect callback, * because the blocked client will be automatically disconnected in * this case, and the user can still hook using the timeout callback. */ bc->disconnect_callback = NULL; } /* Return non-zero if a module command was called in order to fill the * reply for a blocked client. */ int VM_IsBlockedReplyRequest(ValkeyModuleCtx *ctx) { return (ctx->flags & VALKEYMODULE_CTX_BLOCKED_REPLY) != 0; } /* Return non-zero if a module command was called in order to fill the * reply for a blocked client that timed out. */ int VM_IsBlockedTimeoutRequest(ValkeyModuleCtx *ctx) { return (ctx->flags & VALKEYMODULE_CTX_BLOCKED_TIMEOUT) != 0; } /* Get the private data set by ValkeyModule_UnblockClient() */ void *VM_GetBlockedClientPrivateData(ValkeyModuleCtx *ctx) { return ctx->blocked_privdata; } /* Get the key that is ready when the reply callback is called in the context * of a client blocked by ValkeyModule_BlockClientOnKeys(). */ ValkeyModuleString *VM_GetBlockedClientReadyKey(ValkeyModuleCtx *ctx) { return ctx->blocked_ready_key; } /* Get the blocked client associated with a given context. * This is useful in the reply and timeout callbacks of blocked clients, * before sometimes the module has the blocked client handle references * around, and wants to cleanup it. */ ValkeyModuleBlockedClient *VM_GetBlockedClientHandle(ValkeyModuleCtx *ctx) { return ctx->blocked_client; } /* Return true if when the free callback of a blocked client is called, * the reason for the client to be unblocked is that it disconnected * while it was blocked. */ int VM_BlockedClientDisconnected(ValkeyModuleCtx *ctx) { return (ctx->flags & VALKEYMODULE_CTX_BLOCKED_DISCONNECTED) != 0; } /* -------------------------------------------------------------------------- * ## Thread Safe Contexts * -------------------------------------------------------------------------- */ /* Return a context which can be used inside threads to make calls requiring a * context with certain modules APIs. If 'bc' is not NULL then the module will * be bound to a blocked client, and it will be possible to use the * `ValkeyModule_Reply*` family of functions to accumulate a reply for when the * client will be unblocked. Otherwise the thread safe context will be * detached by a specific client. * * To call non-reply APIs, the thread safe context must be prepared with: * * ValkeyModule_ThreadSafeContextLock(ctx); * ... make your call here ... * ValkeyModule_ThreadSafeContextUnlock(ctx); * * This is not needed when using `ValkeyModule_Reply*` functions, assuming * that a blocked client was used when the context was created, otherwise * no ValkeyModule_Reply* call should be made at all. * * NOTE: If you're creating a detached thread safe context (bc is NULL), * consider using `VM_GetDetachedThreadSafeContext` which will also retain * the module ID and thus be more useful for logging. */ ValkeyModuleCtx *VM_GetThreadSafeContext(ValkeyModuleBlockedClient *bc) { ValkeyModuleCtx *ctx = zmalloc(sizeof(*ctx)); ValkeyModule *module = bc ? bc->module : NULL; int flags = VALKEYMODULE_CTX_THREAD_SAFE; /* Creating a new client object is costly. To avoid that, we have an * internal pool of client objects. In blockClient(), a client object is * assigned to bc->thread_safe_ctx_client to be used for the thread safe * context. * For detached thread safe contexts, we create a new client object. * Otherwise, as this function can be called from different threads, we * would need to synchronize access to internal pool of client objects. * Assuming creating detached context is rare and not that performance * critical, we avoid synchronizing access to the client pool by creating * a new client */ if (!bc) flags |= VALKEYMODULE_CTX_NEW_CLIENT; moduleCreateContext(ctx, module, flags); /* Even when the context is associated with a blocked client, we can't * access it safely from another thread, so we use a fake client here * in order to keep things like the currently selected database and similar * things. */ if (bc) { ctx->blocked_client = bc; ctx->client = bc->thread_safe_ctx_client; selectDb(ctx->client, bc->dbid); if (bc->client) { ctx->client->id = bc->client->id; ctx->client->resp = bc->client->resp; } } return ctx; } /* Return a detached thread safe context that is not associated with any * specific blocked client, but is associated with the module's context. * * This is useful for modules that wish to hold a global context over * a long term, for purposes such as logging. */ ValkeyModuleCtx *VM_GetDetachedThreadSafeContext(ValkeyModuleCtx *ctx) { ValkeyModuleCtx *new_ctx = zmalloc(sizeof(*new_ctx)); /* We create a new client object for the detached context. * See VM_GetThreadSafeContext() for more information */ moduleCreateContext(new_ctx, ctx->module, VALKEYMODULE_CTX_THREAD_SAFE | VALKEYMODULE_CTX_NEW_CLIENT); return new_ctx; } /* Release a thread safe context. */ void VM_FreeThreadSafeContext(ValkeyModuleCtx *ctx) { moduleFreeContext(ctx); zfree(ctx); } void moduleGILAfterLock(void) { /* We should never get here if we already inside a module * code block which already opened a context. */ serverAssert(server.execution_nesting == 0); /* Bump up the nesting level to prevent immediate propagation * of possible VM_Call from th thread */ enterExecutionUnit(1, 0); } /* Acquire the server lock before executing a thread safe API call. * This is not needed for `ValkeyModule_Reply*` calls when there is * a blocked client connected to the thread safe context. */ void VM_ThreadSafeContextLock(ValkeyModuleCtx *ctx) { UNUSED(ctx); moduleAcquireGIL(); moduleGILAfterLock(); } /* Similar to VM_ThreadSafeContextLock but this function * would not block if the server lock is already acquired. * * If successful (lock acquired) VALKEYMODULE_OK is returned, * otherwise VALKEYMODULE_ERR is returned and errno is set * accordingly. */ int VM_ThreadSafeContextTryLock(ValkeyModuleCtx *ctx) { UNUSED(ctx); int res = moduleTryAcquireGIL(); if (res != 0) { errno = res; return VALKEYMODULE_ERR; } moduleGILAfterLock(); return VALKEYMODULE_OK; } void moduleGILBeforeUnlock(void) { /* We should never get here if we already inside a module * code block which already opened a context, except * the bump-up from moduleGILAcquired. */ serverAssert(server.execution_nesting == 1); /* Restore nesting level and propagate pending commands * (because it's unclear when thread safe contexts are * released we have to propagate here). */ exitExecutionUnit(); postExecutionUnitOperations(); } /* Release the server lock after a thread safe API call was executed. */ void VM_ThreadSafeContextUnlock(ValkeyModuleCtx *ctx) { UNUSED(ctx); moduleGILBeforeUnlock(); moduleReleaseGIL(); } void moduleAcquireGIL(void) { pthread_mutex_lock(&moduleGIL); } int moduleTryAcquireGIL(void) { return pthread_mutex_trylock(&moduleGIL); } void moduleReleaseGIL(void) { pthread_mutex_unlock(&moduleGIL); } /* -------------------------------------------------------------------------- * ## Module Keyspace Notifications API * -------------------------------------------------------------------------- */ /* Subscribe to keyspace notifications. This is a low-level version of the * keyspace-notifications API. A module can register callbacks to be notified * when keyspace events occur. * * Notification events are filtered by their type (string events, set events, * etc), and the subscriber callback receives only events that match a specific * mask of event types. * * When subscribing to notifications with ValkeyModule_SubscribeToKeyspaceEvents * the module must provide an event type-mask, denoting the events the subscriber * is interested in. This can be an ORed mask of any of the following flags: * * - VALKEYMODULE_NOTIFY_GENERIC: Generic commands like DEL, EXPIRE, RENAME * - VALKEYMODULE_NOTIFY_STRING: String events * - VALKEYMODULE_NOTIFY_LIST: List events * - VALKEYMODULE_NOTIFY_SET: Set events * - VALKEYMODULE_NOTIFY_HASH: Hash events * - VALKEYMODULE_NOTIFY_ZSET: Sorted Set events * - VALKEYMODULE_NOTIFY_EXPIRED: Expiration events * - VALKEYMODULE_NOTIFY_EVICTED: Eviction events * - VALKEYMODULE_NOTIFY_STREAM: Stream events * - VALKEYMODULE_NOTIFY_MODULE: Module types events * - VALKEYMODULE_NOTIFY_KEYMISS: Key-miss events * Notice, key-miss event is the only type * of event that is fired from within a read command. * Performing VM_Call with a write command from within * this notification is wrong and discourage. It will * cause the read command that trigger the event to be * replicated to the AOF/Replica. * - VALKEYMODULE_NOTIFY_ALL: All events (Excluding VALKEYMODULE_NOTIFY_KEYMISS) * - VALKEYMODULE_NOTIFY_LOADED: A special notification available only for modules, * indicates that the key was loaded from persistence. * Notice, when this event fires, the given key * can not be retained, use VM_CreateStringFromString * instead. * * We do not distinguish between key events and keyspace events, and it is up * to the module to filter the actions taken based on the key. * * The subscriber signature is: * * int (*ValkeyModuleNotificationFunc) (ValkeyModuleCtx *ctx, int type, * const char *event, * ValkeyModuleString *key); * * `type` is the event type bit, that must match the mask given at registration * time. The event string is the actual command being executed, and key is the * relevant key. * * Notification callback gets executed with a context that can not be * used to send anything to the client, and has the db number where the event * occurred as its selected db number. * * Notice that it is not necessary to enable notifications in valkey.conf for * module notifications to work. * * Warning: the notification callbacks are performed in a synchronous manner, * so notification callbacks must to be fast, or they would slow the server down. * If you need to take long actions, use threads to offload them. * * Moreover, the fact that the notification is executed synchronously means * that the notification code will be executed in the middle of server logic * (commands logic, eviction, expire). Changing the key space while the logic * runs is dangerous and discouraged. In order to react to key space events with * write actions, please refer to `VM_AddPostNotificationJob`. * * See https://valkey.io/topics/notifications for more information. */ int VM_SubscribeToKeyspaceEvents(ValkeyModuleCtx *ctx, int types, ValkeyModuleNotificationFunc callback) { ValkeyModuleKeyspaceSubscriber *sub = zmalloc(sizeof(*sub)); sub->module = ctx->module; sub->event_mask = types; sub->notify_callback = callback; sub->active = 0; listAddNodeTail(moduleKeyspaceSubscribers, sub); return VALKEYMODULE_OK; } void firePostExecutionUnitJobs(void) { /* Avoid propagation of commands. * In that way, postExecutionUnitOperations will prevent * recursive calls to firePostExecutionUnitJobs. * This is a special case where we need to increase 'execution_nesting' * but we do not want to update the cached time */ enterExecutionUnit(0, 0); while (listLength(modulePostExecUnitJobs) > 0) { listNode *ln = listFirst(modulePostExecUnitJobs); ValkeyModulePostExecUnitJob *job = listNodeValue(ln); listDelNode(modulePostExecUnitJobs, ln); ValkeyModuleCtx ctx; moduleCreateContext(&ctx, job->module, VALKEYMODULE_CTX_TEMP_CLIENT); selectDb(ctx.client, job->dbid); job->callback(&ctx, job->pd); if (job->free_pd) job->free_pd(job->pd); moduleFreeContext(&ctx); zfree(job); } exitExecutionUnit(); } /* When running inside a key space notification callback, it is dangerous and highly discouraged to perform any write * operation (See `VM_SubscribeToKeyspaceEvents`). In order to still perform write actions in this scenario, * the server provides `VM_AddPostNotificationJob` API. The API allows to register a job callback which the server will * call when the following condition are promised to be fulfilled: * 1. It is safe to perform any write operation. * 2. The job will be called atomically along side the key space notification. * * Notice, one job might trigger key space notifications that will trigger more jobs. * This raises a concerns of entering an infinite loops, we consider infinite loops * as a logical bug that need to be fixed in the module, an attempt to protect against * infinite loops by halting the execution could result in violation of the feature correctness * and so the server will make no attempt to protect the module from infinite loops. * * 'free_pd' can be NULL and in such case will not be used. * * Return VALKEYMODULE_OK on success and VALKEYMODULE_ERR if was called while loading data from disk (AOF or RDB) or * if the instance is a readonly replica. */ int VM_AddPostNotificationJob(ValkeyModuleCtx *ctx, ValkeyModulePostNotificationJobFunc callback, void *privdata, void (*free_privdata)(void *)) { if (server.loading || (server.primary_host && server.repl_replica_ro)) { return VALKEYMODULE_ERR; } ValkeyModulePostExecUnitJob *job = zmalloc(sizeof(*job)); job->module = ctx->module; job->callback = callback; job->pd = privdata; job->free_pd = free_privdata; job->dbid = ctx->client->db->id; listAddNodeTail(modulePostExecUnitJobs, job); return VALKEYMODULE_OK; } /* Get the configured bitmap of notify-keyspace-events (Could be used * for additional filtering in ValkeyModuleNotificationFunc) */ int VM_GetNotifyKeyspaceEvents(void) { return server.notify_keyspace_events; } /* Expose notifyKeyspaceEvent to modules */ int VM_NotifyKeyspaceEvent(ValkeyModuleCtx *ctx, int type, const char *event, ValkeyModuleString *key) { if (!ctx || !ctx->client) return VALKEYMODULE_ERR; notifyKeyspaceEvent(type, (char *)event, key, ctx->client->db->id); return VALKEYMODULE_OK; } /* Dispatcher for keyspace notifications to module subscriber functions. * This gets called only if at least one module requested to be notified on * keyspace notifications */ void moduleNotifyKeyspaceEvent(int type, const char *event, robj *key, int dbid) { /* Don't do anything if there aren't any subscribers */ if (listLength(moduleKeyspaceSubscribers) == 0) return; /* Ugly hack to handle modules which use write commands from within * notify_callback, which they should NOT do! * Modules should use ValkeyModules_AddPostNotificationJob instead. * * Anyway, we want any propagated commands from within notify_callback * to be propagated inside a MULTI/EXEC together with the original * command that caused the KSN. * Note that it's only relevant for KSNs which are not generated from within * call(), for example active-expiry and eviction (because anyway * execution_nesting is incremented from within call()) * * In order to do that we increment the execution_nesting counter, thus * preventing postExecutionUnitOperations (from within moduleFreeContext) * from propagating commands from CB. * * This is a special case where we need to increase 'execution_nesting' * but we do not want to update the cached time */ enterExecutionUnit(0, 0); listIter li; listNode *ln; listRewind(moduleKeyspaceSubscribers, &li); /* Remove irrelevant flags from the type mask */ type &= ~(NOTIFY_KEYEVENT | NOTIFY_KEYSPACE); while ((ln = listNext(&li))) { ValkeyModuleKeyspaceSubscriber *sub = ln->value; /* Only notify subscribers on events matching the registration, * and avoid subscribers triggering themselves */ if ((sub->event_mask & type) && (sub->active == 0 || (sub->module->options & VALKEYMODULE_OPTIONS_ALLOW_NESTED_KEYSPACE_NOTIFICATIONS))) { ValkeyModuleCtx ctx; moduleCreateContext(&ctx, sub->module, VALKEYMODULE_CTX_TEMP_CLIENT); selectDb(ctx.client, dbid); /* mark the handler as active to avoid reentrant loops. * If the subscriber performs an action triggering itself, * it will not be notified about it. */ int prev_active = sub->active; sub->active = 1; server.lazy_expire_disabled++; sub->notify_callback(&ctx, type, event, key); server.lazy_expire_disabled--; sub->active = prev_active; moduleFreeContext(&ctx); } } exitExecutionUnit(); } /* Unsubscribe any notification subscribers this module has upon unloading */ void moduleUnsubscribeNotifications(ValkeyModule *module) { listIter li; listNode *ln; listRewind(moduleKeyspaceSubscribers, &li); while ((ln = listNext(&li))) { ValkeyModuleKeyspaceSubscriber *sub = ln->value; if (sub->module == module) { listDelNode(moduleKeyspaceSubscribers, ln); zfree(sub); } } } /* -------------------------------------------------------------------------- * ## Modules Cluster API * -------------------------------------------------------------------------- */ /* The Cluster message callback function pointer type. */ typedef void (*ValkeyModuleClusterMessageReceiver)(ValkeyModuleCtx *ctx, const char *sender_id, uint8_t type, const unsigned char *payload, uint32_t len); /* This structure identifies a registered caller: it must match a given module * ID, for a given message type. The callback function is just the function * that was registered as receiver. */ typedef struct moduleClusterReceiver { uint64_t module_id; ValkeyModuleClusterMessageReceiver callback; struct ValkeyModule *module; struct moduleClusterReceiver *next; } moduleClusterReceiver; typedef struct moduleClusterNodeInfo { int flags; char ip[NET_IP_STR_LEN]; int port; char primary_id[40]; /* Only if flags & VALKEYMODULE_NODE_PRIMARY is true. */ } mdouleClusterNodeInfo; /* We have an array of message types: each bucket is a linked list of * configured receivers. */ static moduleClusterReceiver *clusterReceivers[UINT8_MAX]; /* Dispatch the message to the right module receiver. */ void moduleCallClusterReceivers(const char *sender_id, uint64_t module_id, uint8_t type, const unsigned char *payload, uint32_t len) { moduleClusterReceiver *r = clusterReceivers[type]; while (r) { if (r->module_id == module_id) { ValkeyModuleCtx ctx; moduleCreateContext(&ctx, r->module, VALKEYMODULE_CTX_TEMP_CLIENT); r->callback(&ctx, sender_id, type, payload, len); moduleFreeContext(&ctx); return; } r = r->next; } } /* Register a callback receiver for cluster messages of type 'type'. If there * was already a registered callback, this will replace the callback function * with the one provided, otherwise if the callback is set to NULL and there * is already a callback for this function, the callback is unregistered * (so this API call is also used in order to delete the receiver). */ void VM_RegisterClusterMessageReceiver(ValkeyModuleCtx *ctx, uint8_t type, ValkeyModuleClusterMessageReceiver callback) { if (!server.cluster_enabled) return; uint64_t module_id = moduleTypeEncodeId(ctx->module->name, 0); moduleClusterReceiver *r = clusterReceivers[type], *prev = NULL; while (r) { if (r->module_id == module_id) { /* Found! Set or delete. */ if (callback) { r->callback = callback; } else { /* Delete the receiver entry if the user is setting * it to NULL. Just unlink the receiver node from the * linked list. */ if (prev) prev->next = r->next; else clusterReceivers[type]->next = r->next; zfree(r); } return; } prev = r; r = r->next; } /* Not found, let's add it. */ if (callback) { r = zmalloc(sizeof(*r)); r->module_id = module_id; r->module = ctx->module; r->callback = callback; r->next = clusterReceivers[type]; clusterReceivers[type] = r; } } /* Send a message to all the nodes in the cluster if `target` is NULL, otherwise * at the specified target, which is a VALKEYMODULE_NODE_ID_LEN bytes node ID, as * returned by the receiver callback or by the nodes iteration functions. * * The function returns VALKEYMODULE_OK if the message was successfully sent, * otherwise if the node is not connected or such node ID does not map to any * known cluster node, VALKEYMODULE_ERR is returned. */ int VM_SendClusterMessage(ValkeyModuleCtx *ctx, const char *target_id, uint8_t type, const char *msg, uint32_t len) { if (!server.cluster_enabled) return VALKEYMODULE_ERR; uint64_t module_id = moduleTypeEncodeId(ctx->module->name, 0); if (clusterSendModuleMessageToTarget(target_id, module_id, type, msg, len) == C_OK) return VALKEYMODULE_OK; else return VALKEYMODULE_ERR; } /* Return an array of string pointers, each string pointer points to a cluster * node ID of exactly VALKEYMODULE_NODE_ID_LEN bytes (without any null term). * The number of returned node IDs is stored into `*numnodes`. * However if this function is called by a module not running an an * instance with Cluster enabled, NULL is returned instead. * * The IDs returned can be used with ValkeyModule_GetClusterNodeInfo() in order * to get more information about single node. * * The array returned by this function must be freed using the function * ValkeyModule_FreeClusterNodesList(). * * Example: * * size_t count, j; * char **ids = ValkeyModule_GetClusterNodesList(ctx,&count); * for (j = 0; j < count; j++) { * ValkeyModule_Log(ctx,"notice","Node %.*s", * VALKEYMODULE_NODE_ID_LEN,ids[j]); * } * ValkeyModule_FreeClusterNodesList(ids); */ char **VM_GetClusterNodesList(ValkeyModuleCtx *ctx, size_t *numnodes) { UNUSED(ctx); if (!server.cluster_enabled) return NULL; return getClusterNodesList(numnodes); } /* Free the node list obtained with ValkeyModule_GetClusterNodesList. */ void VM_FreeClusterNodesList(char **ids) { if (ids == NULL) return; for (int j = 0; ids[j]; j++) zfree(ids[j]); zfree(ids); } /* Return this node ID (VALKEYMODULE_CLUSTER_ID_LEN bytes) or NULL if the cluster * is disabled. */ const char *VM_GetMyClusterID(void) { if (!server.cluster_enabled) return NULL; return clusterNodeGetName(getMyClusterNode()); } /* Return the number of nodes in the cluster, regardless of their state * (handshake, noaddress, ...) so that the number of active nodes may actually * be smaller, but not greater than this number. If the instance is not in * cluster mode, zero is returned. */ size_t VM_GetClusterSize(void) { if (!server.cluster_enabled) return 0; return getClusterSize(); } int moduleGetClusterNodeInfoForClient(ValkeyModuleCtx *ctx, client *c, const char *node_id, char *ip, char *primary_id, int *port, int *flags); /* Populate the specified info for the node having as ID the specified 'id', * then returns VALKEYMODULE_OK. Otherwise if the format of node ID is invalid * or the node ID does not exist from the POV of this local node, VALKEYMODULE_ERR * is returned. * * The arguments `ip`, `primary_id`, `port` and `flags` can be NULL in case we don't * need to populate back certain info. If an `ip` and `primary_id` (only populated * if the instance is a replica) are specified, they point to buffers holding * at least VALKEYMODULE_NODE_ID_LEN bytes. The strings written back as `ip` * and `primary_id` are not null terminated. * * The list of flags reported is the following: * * * VALKEYMODULE_NODE_MYSELF: This node * * VALKEYMODULE_NODE_PRIMARY: The node is a primary * * VALKEYMODULE_NODE_REPLICA: The node is a replica * * VALKEYMODULE_NODE_PFAIL: We see the node as failing * * VALKEYMODULE_NODE_FAIL: The cluster agrees the node is failing * * VALKEYMODULE_NODE_NOFAILOVER: The replica is configured to never failover */ int VM_GetClusterNodeInfo(ValkeyModuleCtx *ctx, const char *id, char *ip, char *primary_id, int *port, int *flags) { return moduleGetClusterNodeInfoForClient(ctx, NULL, id, ip, primary_id, port, flags); } /* Like VM_GetClusterNodeInfo(), but returns IP address specifically for the given * client, depending on whether the client is connected over IPv4 or IPv6. * * See also VM_GetClientId(). */ int VM_GetClusterNodeInfoForClient(ValkeyModuleCtx *ctx, uint64_t client_id, const char *node_id, char *ip, char *primary_id, int *port, int *flags) { client *c = lookupClientByID(client_id); if (c == NULL) return VALKEYMODULE_ERR; return moduleGetClusterNodeInfoForClient(ctx, c, node_id, ip, primary_id, port, flags); } int moduleGetClusterNodeInfoForClient(ValkeyModuleCtx *ctx, client *c, const char *node_id, char *ip, char *primary_id, int *port, int *flags) { UNUSED(ctx); clusterNode *node = clusterLookupNode(node_id, strlen(node_id)); if (node == NULL || clusterNodePending(node)) { return VALKEYMODULE_ERR; } if (ip) valkey_strlcpy(ip, clusterNodeIp(node, c), NET_IP_STR_LEN); if (primary_id) { /* If the information is not available, the function will set the * field to zero bytes, so that when the field can't be populated the * function kinda remains predictable. */ if (clusterNodeIsReplica(node) && clusterNodeGetPrimary(node)) memcpy(primary_id, clusterNodeGetName(clusterNodeGetPrimary(node)), VALKEYMODULE_NODE_ID_LEN); else memset(primary_id, 0, VALKEYMODULE_NODE_ID_LEN); } if (port) *port = getNodeDefaultClientPort(node); /* As usually we have to remap flags for modules, in order to ensure * we can provide binary compatibility. */ if (flags) { *flags = 0; if (clusterNodeIsMyself(node)) *flags |= VALKEYMODULE_NODE_MYSELF; if (clusterNodeIsPrimary(node)) *flags |= VALKEYMODULE_NODE_PRIMARY; if (clusterNodeIsReplica(node)) *flags |= VALKEYMODULE_NODE_REPLICA; if (clusterNodeTimedOut(node)) *flags |= VALKEYMODULE_NODE_PFAIL; if (clusterNodeIsFailing(node)) *flags |= VALKEYMODULE_NODE_FAIL; if (clusterNodeIsNoFailover(node)) *flags |= VALKEYMODULE_NODE_NOFAILOVER; } return VALKEYMODULE_OK; } /* Set Cluster flags in order to change the normal behavior of * Cluster, especially with the goal of disabling certain functions. * This is useful for modules that use the Cluster API in order to create * a different distributed system, but still want to use the Cluster * message bus. Flags that can be set: * * * CLUSTER_MODULE_FLAG_NO_FAILOVER * * CLUSTER_MODULE_FLAG_NO_REDIRECTION * * With the following effects: * * * NO_FAILOVER: prevent Cluster replicas from failing over a dead primary. * Also disables the replica migration feature. * * * NO_REDIRECTION: Every node will accept any key, without trying to perform * partitioning according to the Cluster algorithm. * Slots information will still be propagated across the * cluster, but without effect. */ void VM_SetClusterFlags(ValkeyModuleCtx *ctx, uint64_t flags) { UNUSED(ctx); if (flags & VALKEYMODULE_CLUSTER_FLAG_NO_FAILOVER) server.cluster_module_flags |= CLUSTER_MODULE_FLAG_NO_FAILOVER; if (flags & VALKEYMODULE_CLUSTER_FLAG_NO_REDIRECTION) server.cluster_module_flags |= CLUSTER_MODULE_FLAG_NO_REDIRECTION; } /* Returns the cluster slot of a key, similar to the `CLUSTER KEYSLOT` command. * This function works even if cluster mode is not enabled. */ unsigned int VM_ClusterKeySlot(ValkeyModuleString *key) { return keyHashSlot(key->ptr, sdslen(key->ptr)); } /* Returns a short string that can be used as a key or as a hash tag in a key, * such that the key maps to the given cluster slot. Returns NULL if slot is not * a valid slot. */ const char *VM_ClusterCanonicalKeyNameInSlot(unsigned int slot) { return (slot < CLUSTER_SLOTS) ? crc16_slot_table[slot] : NULL; } /* -------------------------------------------------------------------------- * ## Modules Timers API * * Module timers are a high precision "green timers" abstraction where * every module can register even millions of timers without problems, even if * the actual event loop will just have a single timer that is used to awake the * module timers subsystem in order to process the next event. * * All the timers are stored into a radix tree, ordered by expire time, when * the main server event loop timer callback is called, we try to process all * the timers already expired one after the other. Then we re-enter the event * loop registering a timer that will expire when the next to process module * timer will expire. * * Every time the list of active timers drops to zero, we unregister the * main event loop timer, so that there is no overhead when such feature is * not used. * -------------------------------------------------------------------------- */ static rax *Timers; /* The radix tree of all the timers sorted by expire. */ long long aeTimer = -1; /* Main event loop (ae.c) timer identifier. */ typedef void (*ValkeyModuleTimerProc)(ValkeyModuleCtx *ctx, void *data); /* The timer descriptor, stored as value in the radix tree. */ typedef struct ValkeyModuleTimer { ValkeyModule *module; /* Module reference. */ ValkeyModuleTimerProc callback; /* The callback to invoke on expire. */ void *data; /* Private data for the callback. */ int dbid; /* Database number selected by the original client. */ } ValkeyModuleTimer; /* This is the timer handler that is called by the main event loop. We schedule * this timer to be called when the nearest of our module timers will expire. */ long long moduleTimerHandler(struct aeEventLoop *eventLoop, long long id, void *clientData) { UNUSED(eventLoop); UNUSED(id); UNUSED(clientData); /* To start let's try to fire all the timers already expired. */ raxIterator ri; raxStart(&ri, Timers); uint64_t now = ustime(); long long next_period = 0; while (1) { raxSeek(&ri, "^", NULL, 0); if (!raxNext(&ri)) break; uint64_t expiretime; memcpy(&expiretime, ri.key, sizeof(expiretime)); expiretime = ntohu64(expiretime); if (now >= expiretime) { ValkeyModuleTimer *timer = ri.data; ValkeyModuleCtx ctx; moduleCreateContext(&ctx, timer->module, VALKEYMODULE_CTX_TEMP_CLIENT); selectDb(ctx.client, timer->dbid); timer->callback(&ctx, timer->data); moduleFreeContext(&ctx); raxRemove(Timers, (unsigned char *)ri.key, ri.key_len, NULL); zfree(timer); } else { /* We call ustime() again instead of using the cached 'now' so that * 'next_period' isn't affected by the time it took to execute * previous calls to 'callback. * We need to cast 'expiretime' so that the compiler will not treat * the difference as unsigned (Causing next_period to be huge) in * case expiretime < ustime() */ next_period = ((long long)expiretime - ustime()) / 1000; /* Scale to milliseconds. */ break; } } raxStop(&ri); /* Reschedule the next timer or cancel it. */ if (next_period <= 0) next_period = 1; if (raxSize(Timers) > 0) { return next_period; } else { aeTimer = -1; return AE_NOMORE; } } /* Create a new timer that will fire after `period` milliseconds, and will call * the specified function using `data` as argument. The returned timer ID can be * used to get information from the timer or to stop it before it fires. * Note that for the common use case of a repeating timer (Re-registration * of the timer inside the ValkeyModuleTimerProc callback) it matters when * this API is called: * If it is called at the beginning of 'callback' it means * the event will triggered every 'period'. * If it is called at the end of 'callback' it means * there will 'period' milliseconds gaps between events. * (If the time it takes to execute 'callback' is negligible the two * statements above mean the same) */ ValkeyModuleTimerID VM_CreateTimer(ValkeyModuleCtx *ctx, mstime_t period, ValkeyModuleTimerProc callback, void *data) { ValkeyModuleTimer *timer = zmalloc(sizeof(*timer)); timer->module = ctx->module; timer->callback = callback; timer->data = data; timer->dbid = ctx->client ? ctx->client->db->id : 0; uint64_t expiretime = ustime() + period * 1000; uint64_t key; while (1) { key = htonu64(expiretime); if (!raxFind(Timers, (unsigned char *)&key, sizeof(key), NULL)) { raxInsert(Timers, (unsigned char *)&key, sizeof(key), timer, NULL); break; } else { expiretime++; } } /* We need to install the main event loop timer if it's not already * installed, or we may need to refresh its period if we just installed * a timer that will expire sooner than any other else (i.e. the timer * we just installed is the first timer in the Timers rax). */ if (aeTimer != -1) { raxIterator ri; raxStart(&ri, Timers); raxSeek(&ri, "^", NULL, 0); raxNext(&ri); if (memcmp(ri.key, &key, sizeof(key)) == 0) { /* This is the first key, we need to re-install the timer according * to the just added event. */ aeDeleteTimeEvent(server.el, aeTimer); aeTimer = -1; } raxStop(&ri); } /* If we have no main timer (the old one was invalidated, or this is the * first module timer we have), install one. */ if (aeTimer == -1) aeTimer = aeCreateTimeEvent(server.el, period, moduleTimerHandler, NULL, NULL); return key; } /* Stop a timer, returns VALKEYMODULE_OK if the timer was found, belonged to the * calling module, and was stopped, otherwise VALKEYMODULE_ERR is returned. * If not NULL, the data pointer is set to the value of the data argument when * the timer was created. */ int VM_StopTimer(ValkeyModuleCtx *ctx, ValkeyModuleTimerID id, void **data) { void *result; if (!raxFind(Timers, (unsigned char *)&id, sizeof(id), &result)) return VALKEYMODULE_ERR; ValkeyModuleTimer *timer = result; if (timer->module != ctx->module) return VALKEYMODULE_ERR; if (data) *data = timer->data; raxRemove(Timers, (unsigned char *)&id, sizeof(id), NULL); zfree(timer); return VALKEYMODULE_OK; } /* Obtain information about a timer: its remaining time before firing * (in milliseconds), and the private data pointer associated with the timer. * If the timer specified does not exist or belongs to a different module * no information is returned and the function returns VALKEYMODULE_ERR, otherwise * VALKEYMODULE_OK is returned. The arguments remaining or data can be NULL if * the caller does not need certain information. */ int VM_GetTimerInfo(ValkeyModuleCtx *ctx, ValkeyModuleTimerID id, uint64_t *remaining, void **data) { void *result; if (!raxFind(Timers, (unsigned char *)&id, sizeof(id), &result)) return VALKEYMODULE_ERR; ValkeyModuleTimer *timer = result; if (timer->module != ctx->module) return VALKEYMODULE_ERR; if (remaining) { int64_t rem = ntohu64(id) - ustime(); if (rem < 0) rem = 0; *remaining = rem / 1000; /* Scale to milliseconds. */ } if (data) *data = timer->data; return VALKEYMODULE_OK; } /* Query timers to see if any timer belongs to the module. * Return 1 if any timer was found, otherwise 0 would be returned. */ int moduleHoldsTimer(struct ValkeyModule *module) { raxIterator iter; int found = 0; raxStart(&iter, Timers); raxSeek(&iter, "^", NULL, 0); while (raxNext(&iter)) { ValkeyModuleTimer *timer = iter.data; if (timer->module == module) { found = 1; break; } } raxStop(&iter); return found; } /* -------------------------------------------------------------------------- * ## Modules EventLoop API * --------------------------------------------------------------------------*/ typedef struct EventLoopData { ValkeyModuleEventLoopFunc rFunc; ValkeyModuleEventLoopFunc wFunc; void *user_data; } EventLoopData; typedef struct EventLoopOneShot { ValkeyModuleEventLoopOneShotFunc func; void *user_data; } EventLoopOneShot; list *moduleEventLoopOneShots; static pthread_mutex_t moduleEventLoopMutex = PTHREAD_MUTEX_INITIALIZER; static int eventLoopToAeMask(int mask) { int aeMask = 0; if (mask & VALKEYMODULE_EVENTLOOP_READABLE) aeMask |= AE_READABLE; if (mask & VALKEYMODULE_EVENTLOOP_WRITABLE) aeMask |= AE_WRITABLE; return aeMask; } static int eventLoopFromAeMask(int ae_mask) { int mask = 0; if (ae_mask & AE_READABLE) mask |= VALKEYMODULE_EVENTLOOP_READABLE; if (ae_mask & AE_WRITABLE) mask |= VALKEYMODULE_EVENTLOOP_WRITABLE; return mask; } static void eventLoopCbReadable(struct aeEventLoop *ae, int fd, void *user_data, int ae_mask) { UNUSED(ae); EventLoopData *data = user_data; data->rFunc(fd, data->user_data, eventLoopFromAeMask(ae_mask)); } static void eventLoopCbWritable(struct aeEventLoop *ae, int fd, void *user_data, int ae_mask) { UNUSED(ae); EventLoopData *data = user_data; data->wFunc(fd, data->user_data, eventLoopFromAeMask(ae_mask)); } /* Add a pipe / socket event to the event loop. * * * `mask` must be one of the following values: * * * `VALKEYMODULE_EVENTLOOP_READABLE` * * `VALKEYMODULE_EVENTLOOP_WRITABLE` * * `VALKEYMODULE_EVENTLOOP_READABLE | VALKEYMODULE_EVENTLOOP_WRITABLE` * * On success VALKEYMODULE_OK is returned, otherwise * VALKEYMODULE_ERR is returned and errno is set to the following values: * * * ERANGE: `fd` is negative or higher than `maxclients` server config. * * EINVAL: `callback` is NULL or `mask` value is invalid. * * `errno` might take other values in case of an internal error. * * Example: * * void onReadable(int fd, void *user_data, int mask) { * char buf[32]; * int bytes = read(fd,buf,sizeof(buf)); * printf("Read %d bytes \n", bytes); * } * VM_EventLoopAdd(fd, VALKEYMODULE_EVENTLOOP_READABLE, onReadable, NULL); */ int VM_EventLoopAdd(int fd, int mask, ValkeyModuleEventLoopFunc func, void *user_data) { if (fd < 0 || fd >= aeGetSetSize(server.el)) { errno = ERANGE; return VALKEYMODULE_ERR; } if (!func || mask & ~(VALKEYMODULE_EVENTLOOP_READABLE | VALKEYMODULE_EVENTLOOP_WRITABLE)) { errno = EINVAL; return VALKEYMODULE_ERR; } /* We are going to register stub callbacks to 'ae' for two reasons: * * - "ae" callback signature is different from ValkeyModuleEventLoopCallback, * that will be handled it in our stub callbacks. * - We need to remap 'mask' value to provide binary compatibility. * * For the stub callbacks, saving user 'callback' and 'user_data' in an * EventLoopData object and passing it to ae, later, we'll extract * 'callback' and 'user_data' from that. */ EventLoopData *data = aeGetFileClientData(server.el, fd); if (!data) data = zcalloc(sizeof(*data)); aeFileProc *aeProc; if (mask & VALKEYMODULE_EVENTLOOP_READABLE) aeProc = eventLoopCbReadable; else aeProc = eventLoopCbWritable; int aeMask = eventLoopToAeMask(mask); if (aeCreateFileEvent(server.el, fd, aeMask, aeProc, data) != AE_OK) { if (aeGetFileEvents(server.el, fd) == AE_NONE) zfree(data); return VALKEYMODULE_ERR; } data->user_data = user_data; if (mask & VALKEYMODULE_EVENTLOOP_READABLE) data->rFunc = func; if (mask & VALKEYMODULE_EVENTLOOP_WRITABLE) data->wFunc = func; errno = 0; return VALKEYMODULE_OK; } /* Delete a pipe / socket event from the event loop. * * * `mask` must be one of the following values: * * * `VALKEYMODULE_EVENTLOOP_READABLE` * * `VALKEYMODULE_EVENTLOOP_WRITABLE` * * `VALKEYMODULE_EVENTLOOP_READABLE | VALKEYMODULE_EVENTLOOP_WRITABLE` * * On success VALKEYMODULE_OK is returned, otherwise * VALKEYMODULE_ERR is returned and errno is set to the following values: * * * ERANGE: `fd` is negative or higher than `maxclients` server config. * * EINVAL: `mask` value is invalid. */ int VM_EventLoopDel(int fd, int mask) { if (fd < 0 || fd >= aeGetSetSize(server.el)) { errno = ERANGE; return VALKEYMODULE_ERR; } if (mask & ~(VALKEYMODULE_EVENTLOOP_READABLE | VALKEYMODULE_EVENTLOOP_WRITABLE)) { errno = EINVAL; return VALKEYMODULE_ERR; } /* After deleting the event, if fd does not have any registered event * anymore, we can free the EventLoopData object. */ EventLoopData *data = aeGetFileClientData(server.el, fd); aeDeleteFileEvent(server.el, fd, eventLoopToAeMask(mask)); if (aeGetFileEvents(server.el, fd) == AE_NONE) zfree(data); errno = 0; return VALKEYMODULE_OK; } /* This function can be called from other threads to trigger callback on the server * main thread. On success VALKEYMODULE_OK is returned. If `func` is NULL * VALKEYMODULE_ERR is returned and errno is set to EINVAL. */ int VM_EventLoopAddOneShot(ValkeyModuleEventLoopOneShotFunc func, void *user_data) { if (!func) { errno = EINVAL; return VALKEYMODULE_ERR; } EventLoopOneShot *oneshot = zmalloc(sizeof(*oneshot)); oneshot->func = func; oneshot->user_data = user_data; pthread_mutex_lock(&moduleEventLoopMutex); if (!moduleEventLoopOneShots) moduleEventLoopOneShots = listCreate(); listAddNodeTail(moduleEventLoopOneShots, oneshot); pthread_mutex_unlock(&moduleEventLoopMutex); if (write(server.module_pipe[1], "A", 1) != 1) { /* Pipe is non-blocking, write() may fail if it's full. */ } errno = 0; return VALKEYMODULE_OK; } /* This function will check the moduleEventLoopOneShots queue in order to * call the callback for the registered oneshot events. */ static void eventLoopHandleOneShotEvents(void) { pthread_mutex_lock(&moduleEventLoopMutex); if (moduleEventLoopOneShots) { while (listLength(moduleEventLoopOneShots)) { listNode *ln = listFirst(moduleEventLoopOneShots); EventLoopOneShot *oneshot = ln->value; listDelNode(moduleEventLoopOneShots, ln); /* Unlock mutex before the callback. Another oneshot event can be * added in the callback, it will need to lock the mutex. */ pthread_mutex_unlock(&moduleEventLoopMutex); oneshot->func(oneshot->user_data); zfree(oneshot); /* Lock again for the next iteration */ pthread_mutex_lock(&moduleEventLoopMutex); } } pthread_mutex_unlock(&moduleEventLoopMutex); } /* -------------------------------------------------------------------------- * ## Modules ACL API * * Implements a hook into the authentication and authorization within the server. * --------------------------------------------------------------------------*/ /* This function is called when a client's user has changed and invokes the * client's user changed callback if it was set. This callback should * cleanup any state the module was tracking about this client. * * A client's user can be changed through the AUTH command, module * authentication, and when a client is freed. */ void moduleNotifyUserChanged(client *c) { if (!c->module_data || !c->module_data->auth_callback) return; c->module_data->auth_callback(c->id, c->module_data->auth_callback_privdata); /* The callback will fire exactly once, even if the user remains * the same. It is expected to completely clean up the state * so all references are cleared here. */ c->module_data->auth_callback = NULL; c->module_data->auth_callback_privdata = NULL; c->module_data->auth_module = NULL; } void revokeClientAuthentication(client *c) { /* Freeing the client would result in moduleNotifyUserChanged() to be * called later, however since we use revokeClientAuthentication() also * in moduleFreeAuthenticatedClients() to implement module unloading, we * do this action ASAP: this way if the module is unloaded, when the client * is eventually freed we don't rely on the module to still exist. */ moduleNotifyUserChanged(c); c->user = DefaultUser; c->flag.authenticated = 0; /* We will write replies to this client later, so we can't close it * directly even if async. */ if (c == server.current_client) { c->flag.close_after_command = 1; } else { freeClientAsync(c); } } /* Cleanup all clients that have been authenticated with this module. This * is called from onUnload() to give the module a chance to cleanup any * resources associated with clients it has authenticated. */ static void moduleFreeAuthenticatedClients(ValkeyModule *module) { listIter li; listNode *ln; listRewind(server.clients, &li); while ((ln = listNext(&li)) != NULL) { client *c = listNodeValue(ln); if (!c->module_data || !c->module_data->auth_module) continue; ValkeyModule *auth_module = (ValkeyModule *)c->module_data->auth_module; if (auth_module == module) { revokeClientAuthentication(c); } } } /* Creates an ACL user that the module can use to authenticate a client. * After obtaining the user, the module should set what such user can do * using the VM_SetUserACL() function. Once configured, the user * can be used in order to authenticate a connection, with the specified * ACL rules, using the ValkeyModule_AuthClientWithUser() function. * * Note that: * * * Users created here are not listed by the ACL command. * * Users created here are not checked for duplicated name, so it's up to * the module calling this function to take care of not creating users * with the same name. * * The created user can be used to authenticate multiple connections. * * The caller can later free the user using the function * VM_FreeModuleUser(). When this function is called, if there are * still clients authenticated with this user, they are disconnected. * The function to free the user should only be used when the caller really * wants to invalidate the user to define a new one with different * capabilities. */ ValkeyModuleUser *VM_CreateModuleUser(const char *name) { ValkeyModuleUser *new_user = zmalloc(sizeof(ValkeyModuleUser)); new_user->user = ACLCreateUnlinkedUser(); new_user->free_user = 1; /* Free the previous temporarily assigned name to assign the new one */ sdsfree(new_user->user->name); new_user->user->name = sdsnew(name); return new_user; } /* Frees a given user and disconnects all of the clients that have been * authenticated with it. See VM_CreateModuleUser for detailed usage.*/ int VM_FreeModuleUser(ValkeyModuleUser *user) { if (user->free_user) ACLFreeUserAndKillClients(user->user); zfree(user); return VALKEYMODULE_OK; } /* Sets the permissions of a user created through the module * interface. The syntax is the same as ACL SETUSER, so refer to the * documentation in acl.c for more information. See VM_CreateModuleUser * for detailed usage. * * Returns VALKEYMODULE_OK on success and VALKEYMODULE_ERR on failure * and will set an errno describing why the operation failed. */ int VM_SetModuleUserACL(ValkeyModuleUser *user, const char *acl) { return ACLSetUser(user->user, acl, -1); } /* Sets the permission of a user with a complete ACL string, such as one * would use on the ACL SETUSER command line API. This differs from * VM_SetModuleUserACL, which only takes single ACL operations at a time. * * Returns VALKEYMODULE_OK on success and VALKEYMODULE_ERR on failure * if a ValkeyModuleString is provided in error, a string describing the error * will be returned */ int VM_SetModuleUserACLString(ValkeyModuleCtx *ctx, ValkeyModuleUser *user, const char *acl, ValkeyModuleString **error) { serverAssert(user != NULL); int argc; sds *argv = sdssplitargs(acl, &argc); sds err = ACLStringSetUser(user->user, NULL, argv, argc); sdsfreesplitres(argv, argc); if (err) { if (error) { *error = createObject(OBJ_STRING, err); if (ctx != NULL) autoMemoryAdd(ctx, VALKEYMODULE_AM_STRING, *error); } else { sdsfree(err); } return VALKEYMODULE_ERR; } return VALKEYMODULE_OK; } /* Get the ACL string for a given user * Returns a ValkeyModuleString */ ValkeyModuleString *VM_GetModuleUserACLString(ValkeyModuleUser *user) { serverAssert(user != NULL); return ACLDescribeUser(user->user); } /* Retrieve the user name of the client connection behind the current context. * The user name can be used later, in order to get a ValkeyModuleUser. * See more information in VM_GetModuleUserFromUserName. * * The returned string must be released with ValkeyModule_FreeString() or by * enabling automatic memory management. */ ValkeyModuleString *VM_GetCurrentUserName(ValkeyModuleCtx *ctx) { return VM_CreateString(ctx, ctx->client->user->name, sdslen(ctx->client->user->name)); } /* A ValkeyModuleUser can be used to check if command, key or channel can be executed or * accessed according to the ACLs rules associated with that user. * When a Module wants to do ACL checks on a general ACL user (not created by VM_CreateModuleUser), * it can get the ValkeyModuleUser from this API, based on the user name retrieved by VM_GetCurrentUserName. * * Since a general ACL user can be deleted at any time, this ValkeyModuleUser should be used only in the context * where this function was called. In order to do ACL checks out of that context, the Module can store the user name, * and call this API at any other context. * * Returns NULL if the user is disabled or the user does not exist. * The caller should later free the user using the function VM_FreeModuleUser().*/ ValkeyModuleUser *VM_GetModuleUserFromUserName(ValkeyModuleString *name) { /* First, verify that the user exist */ user *acl_user = ACLGetUserByName(name->ptr, sdslen(name->ptr)); if (acl_user == NULL) { return NULL; } ValkeyModuleUser *new_user = zmalloc(sizeof(ValkeyModuleUser)); new_user->user = acl_user; new_user->free_user = 0; return new_user; } /* Checks if the command can be executed by the user, according to the ACLs associated with it. * * On success a VALKEYMODULE_OK is returned, otherwise * VALKEYMODULE_ERR is returned and errno is set to the following values: * * * ENOENT: Specified command does not exist. * * EACCES: Command cannot be executed, according to ACL rules */ int VM_ACLCheckCommandPermissions(ValkeyModuleUser *user, ValkeyModuleString **argv, int argc) { int keyidxptr; struct serverCommand *cmd; /* Find command */ if ((cmd = lookupCommand(argv, argc)) == NULL) { errno = ENOENT; return VALKEYMODULE_ERR; } if (ACLCheckAllUserCommandPerm(user->user, cmd, argv, argc, &keyidxptr) != ACL_OK) { errno = EACCES; return VALKEYMODULE_ERR; } return VALKEYMODULE_OK; } /* Check if the key can be accessed by the user according to the ACLs attached to the user * and the flags representing the key access. The flags are the same that are used in the * keyspec for logical operations. These flags are documented in ValkeyModule_SetCommandInfo as * the VALKEYMODULE_CMD_KEY_ACCESS, VALKEYMODULE_CMD_KEY_UPDATE, VALKEYMODULE_CMD_KEY_INSERT, * and VALKEYMODULE_CMD_KEY_DELETE flags. * * If no flags are supplied, the user is still required to have some access to the key for * this command to return successfully. * * If the user is able to access the key then VALKEYMODULE_OK is returned, otherwise * VALKEYMODULE_ERR is returned and errno is set to one of the following values: * * * EINVAL: The provided flags are invalid. * * EACCESS: The user does not have permission to access the key. */ int VM_ACLCheckKeyPermissions(ValkeyModuleUser *user, ValkeyModuleString *key, int flags) { const int allow_mask = (VALKEYMODULE_CMD_KEY_ACCESS | VALKEYMODULE_CMD_KEY_INSERT | VALKEYMODULE_CMD_KEY_DELETE | VALKEYMODULE_CMD_KEY_UPDATE); if ((flags & allow_mask) != flags) { errno = EINVAL; return VALKEYMODULE_ERR; } int keyspec_flags = moduleConvertKeySpecsFlags(flags, 0); if (ACLUserCheckKeyPerm(user->user, key->ptr, sdslen(key->ptr), keyspec_flags) != ACL_OK) { errno = EACCES; return VALKEYMODULE_ERR; } return VALKEYMODULE_OK; } /* Check if the pubsub channel can be accessed by the user based off of the given * access flags. See VM_ChannelAtPosWithFlags for more information about the * possible flags that can be passed in. * * If the user is able to access the pubsub channel then VALKEYMODULE_OK is returned, otherwise * VALKEYMODULE_ERR is returned and errno is set to one of the following values: * * * EINVAL: The provided flags are invalid. * * EACCESS: The user does not have permission to access the pubsub channel. */ int VM_ACLCheckChannelPermissions(ValkeyModuleUser *user, ValkeyModuleString *ch, int flags) { const int allow_mask = (VALKEYMODULE_CMD_CHANNEL_PUBLISH | VALKEYMODULE_CMD_CHANNEL_SUBSCRIBE | VALKEYMODULE_CMD_CHANNEL_UNSUBSCRIBE | VALKEYMODULE_CMD_CHANNEL_PATTERN); if ((flags & allow_mask) != flags) { errno = EINVAL; return VALKEYMODULE_ERR; } /* Unsubscribe permissions are currently always allowed. */ if (flags & VALKEYMODULE_CMD_CHANNEL_UNSUBSCRIBE) { return VALKEYMODULE_OK; } int is_pattern = flags & VALKEYMODULE_CMD_CHANNEL_PATTERN; if (ACLUserCheckChannelPerm(user->user, ch->ptr, is_pattern) != ACL_OK) return VALKEYMODULE_ERR; return VALKEYMODULE_OK; } /* Helper function to map a ValkeyModuleACLLogEntryReason to ACL Log entry reason. */ int moduleGetACLLogEntryReason(ValkeyModuleACLLogEntryReason reason) { int acl_reason = 0; switch (reason) { case VALKEYMODULE_ACL_LOG_AUTH: acl_reason = ACL_DENIED_AUTH; break; case VALKEYMODULE_ACL_LOG_KEY: acl_reason = ACL_DENIED_KEY; break; case VALKEYMODULE_ACL_LOG_CHANNEL: acl_reason = ACL_DENIED_CHANNEL; break; case VALKEYMODULE_ACL_LOG_CMD: acl_reason = ACL_DENIED_CMD; break; default: break; } return acl_reason; } /* Adds a new entry in the ACL log. * Returns VALKEYMODULE_OK on success and VALKEYMODULE_ERR on error. * * For more information about ACL log, please refer to https://valkey.io/commands/acl-log */ int VM_ACLAddLogEntry(ValkeyModuleCtx *ctx, ValkeyModuleUser *user, ValkeyModuleString *object, ValkeyModuleACLLogEntryReason reason) { int acl_reason = moduleGetACLLogEntryReason(reason); if (!acl_reason) return VALKEYMODULE_ERR; addACLLogEntry(ctx->client, acl_reason, ACL_LOG_CTX_MODULE, -1, user->user->name, sdsdup(object->ptr)); return VALKEYMODULE_OK; } /* Adds a new entry in the ACL log with the `username` ValkeyModuleString provided. * Returns VALKEYMODULE_OK on success and VALKEYMODULE_ERR on error. * * For more information about ACL log, please refer to https://valkey.io/commands/acl-log */ int VM_ACLAddLogEntryByUserName(ValkeyModuleCtx *ctx, ValkeyModuleString *username, ValkeyModuleString *object, ValkeyModuleACLLogEntryReason reason) { int acl_reason = moduleGetACLLogEntryReason(reason); if (!acl_reason) return VALKEYMODULE_ERR; addACLLogEntry(ctx->client, acl_reason, ACL_LOG_CTX_MODULE, -1, username->ptr, sdsdup(object->ptr)); return VALKEYMODULE_OK; } /* Authenticate the client associated with the context with * the provided user. Returns VALKEYMODULE_OK on success and * VALKEYMODULE_ERR on error. * * This authentication can be tracked with the optional callback and private * data fields. The callback will be called whenever the user of the client * changes. This callback should be used to cleanup any state that is being * kept in the module related to the client authentication. It will only be * called once, even when the user hasn't changed, in order to allow for a * new callback to be specified. If this authentication does not need to be * tracked, pass in NULL for the callback and privdata. * * If client_id is not NULL, it will be filled with the id of the client * that was authenticated. This can be used with the * VM_DeauthenticateAndCloseClient() API in order to deauthenticate a * previously authenticated client if the authentication is no longer valid. * * For expensive authentication operations, it is recommended to block the * client and do the authentication in the background and then attach the user * to the client in a threadsafe context. */ static int authenticateClientWithUser(ValkeyModuleCtx *ctx, user *user, ValkeyModuleUserChangedFunc callback, void *privdata, uint64_t *client_id) { if (user->flags & USER_FLAG_DISABLED) { return VALKEYMODULE_ERR; } /* Avoid settings which are meaningless and will be lost */ if (!ctx->client || (ctx->client->flag.module)) { return VALKEYMODULE_ERR; } moduleNotifyUserChanged(ctx->client); ctx->client->user = user; ctx->client->flag.authenticated = 1; if (clientHasModuleAuthInProgress(ctx->client)) { ctx->client->flag.module_auth_has_result = 1; } if (callback) { initClientModuleData(ctx->client); ctx->client->module_data->auth_callback = callback; ctx->client->module_data->auth_callback_privdata = privdata; ctx->client->module_data->auth_module = ctx->module; } if (client_id) { *client_id = ctx->client->id; } return VALKEYMODULE_OK; } /* Authenticate the current context's user with the provided acl user. * Returns VALKEYMODULE_ERR if the user is disabled. * * See authenticateClientWithUser for information about callback, client_id, * and general usage for authentication. */ int VM_AuthenticateClientWithUser(ValkeyModuleCtx *ctx, ValkeyModuleUser *module_user, ValkeyModuleUserChangedFunc callback, void *privdata, uint64_t *client_id) { return authenticateClientWithUser(ctx, module_user->user, callback, privdata, client_id); } /* Authenticate the current context's user with the provided acl user. * Returns VALKEYMODULE_ERR if the user is disabled or the user does not exist. * * See authenticateClientWithUser for information about callback, client_id, * and general usage for authentication. */ int VM_AuthenticateClientWithACLUser(ValkeyModuleCtx *ctx, const char *name, size_t len, ValkeyModuleUserChangedFunc callback, void *privdata, uint64_t *client_id) { user *acl_user = ACLGetUserByName(name, len); if (!acl_user) { return VALKEYMODULE_ERR; } return authenticateClientWithUser(ctx, acl_user, callback, privdata, client_id); } /* Deauthenticate and close the client. The client resources will not be * immediately freed, but will be cleaned up in a background job. This is * the recommended way to deauthenticate a client since most clients can't * handle users becoming deauthenticated. Returns VALKEYMODULE_ERR when the * client doesn't exist and VALKEYMODULE_OK when the operation was successful. * * The client ID is returned from the VM_AuthenticateClientWithUser and * VM_AuthenticateClientWithACLUser APIs, but can be obtained through * the CLIENT api or through server events. * * This function is not thread safe, and must be executed within the context * of a command or thread safe context. */ int VM_DeauthenticateAndCloseClient(ValkeyModuleCtx *ctx, uint64_t client_id) { UNUSED(ctx); client *c = lookupClientByID(client_id); if (c == NULL) return VALKEYMODULE_ERR; /* Revoke also marks client to be closed ASAP */ revokeClientAuthentication(c); return VALKEYMODULE_OK; } /* Redact the client command argument specified at the given position. Redacted arguments * are obfuscated in user facing commands such as SLOWLOG or MONITOR, as well as * never being written to server logs. This command may be called multiple times on the * same position. * * Note that the command name, position 0, can not be redacted. * * Returns VALKEYMODULE_OK if the argument was redacted and VALKEYMODULE_ERR if there * was an invalid parameter passed in or the position is outside the client * argument range. */ int VM_RedactClientCommandArgument(ValkeyModuleCtx *ctx, int pos) { if (!ctx || !ctx->client || pos <= 0 || ctx->client->argc <= pos) { return VALKEYMODULE_ERR; } redactClientCommandArgument(ctx->client, pos); return VALKEYMODULE_OK; } /* Return the X.509 client-side certificate used by the client to authenticate * this connection. * * The return value is an allocated ValkeyModuleString that is a X.509 certificate * encoded in PEM (Base64) format. It should be freed (or auto-freed) by the caller. * * A NULL value is returned in the following conditions: * * - Connection ID does not exist * - Connection is not a TLS connection * - Connection is a TLS connection but no client certificate was used */ ValkeyModuleString *VM_GetClientCertificate(ValkeyModuleCtx *ctx, uint64_t client_id) { client *c = lookupClientByID(client_id); if (c == NULL) return NULL; sds cert = connGetPeerCert(c->conn); if (!cert) return NULL; ValkeyModuleString *s = createObject(OBJ_STRING, cert); if (ctx != NULL) autoMemoryAdd(ctx, VALKEYMODULE_AM_STRING, s); return s; } /* -------------------------------------------------------------------------- * ## Modules Dictionary API * * Implements a sorted dictionary (actually backed by a radix tree) with * the usual get / set / del / num-items API, together with an iterator * capable of going back and forth. * -------------------------------------------------------------------------- */ /* Create a new dictionary. The 'ctx' pointer can be the current module context * or NULL, depending on what you want. Please follow the following rules: * * 1. Use a NULL context if you plan to retain a reference to this dictionary * that will survive the time of the module callback where you created it. * 2. Use a NULL context if no context is available at the time you are creating * the dictionary (of course...). * 3. However use the current callback context as 'ctx' argument if the * dictionary time to live is just limited to the callback scope. In this * case, if enabled, you can enjoy the automatic memory management that will * reclaim the dictionary memory, as well as the strings returned by the * Next / Prev dictionary iterator calls. */ ValkeyModuleDict *VM_CreateDict(ValkeyModuleCtx *ctx) { struct ValkeyModuleDict *d = zmalloc(sizeof(*d)); d->rax = raxNew(); if (ctx != NULL) autoMemoryAdd(ctx, VALKEYMODULE_AM_DICT, d); return d; } /* Free a dictionary created with VM_CreateDict(). You need to pass the * context pointer 'ctx' only if the dictionary was created using the * context instead of passing NULL. */ void VM_FreeDict(ValkeyModuleCtx *ctx, ValkeyModuleDict *d) { if (ctx != NULL) autoMemoryFreed(ctx, VALKEYMODULE_AM_DICT, d); raxFree(d->rax); zfree(d); } /* Return the size of the dictionary (number of keys). */ uint64_t VM_DictSize(ValkeyModuleDict *d) { return raxSize(d->rax); } /* Store the specified key into the dictionary, setting its value to the * pointer 'ptr'. If the key was added with success, since it did not * already exist, VALKEYMODULE_OK is returned. Otherwise if the key already * exists the function returns VALKEYMODULE_ERR. */ int VM_DictSetC(ValkeyModuleDict *d, void *key, size_t keylen, void *ptr) { int retval = raxTryInsert(d->rax, key, keylen, ptr, NULL); return (retval == 1) ? VALKEYMODULE_OK : VALKEYMODULE_ERR; } /* Like ValkeyModule_DictSetC() but will replace the key with the new * value if the key already exists. */ int VM_DictReplaceC(ValkeyModuleDict *d, void *key, size_t keylen, void *ptr) { int retval = raxInsert(d->rax, key, keylen, ptr, NULL); return (retval == 1) ? VALKEYMODULE_OK : VALKEYMODULE_ERR; } /* Like ValkeyModule_DictSetC() but takes the key as a ValkeyModuleString. */ int VM_DictSet(ValkeyModuleDict *d, ValkeyModuleString *key, void *ptr) { return VM_DictSetC(d, key->ptr, sdslen(key->ptr), ptr); } /* Like ValkeyModule_DictReplaceC() but takes the key as a ValkeyModuleString. */ int VM_DictReplace(ValkeyModuleDict *d, ValkeyModuleString *key, void *ptr) { return VM_DictReplaceC(d, key->ptr, sdslen(key->ptr), ptr); } /* Return the value stored at the specified key. The function returns NULL * both in the case the key does not exist, or if you actually stored * NULL at key. So, optionally, if the 'nokey' pointer is not NULL, it will * be set by reference to 1 if the key does not exist, or to 0 if the key * exists. */ void *VM_DictGetC(ValkeyModuleDict *d, void *key, size_t keylen, int *nokey) { void *res = NULL; int found = raxFind(d->rax, key, keylen, &res); if (nokey) *nokey = !found; return res; } /* Like ValkeyModule_DictGetC() but takes the key as a ValkeyModuleString. */ void *VM_DictGet(ValkeyModuleDict *d, ValkeyModuleString *key, int *nokey) { return VM_DictGetC(d, key->ptr, sdslen(key->ptr), nokey); } /* Remove the specified key from the dictionary, returning VALKEYMODULE_OK if * the key was found and deleted, or VALKEYMODULE_ERR if instead there was * no such key in the dictionary. When the operation is successful, if * 'oldval' is not NULL, then '*oldval' is set to the value stored at the * key before it was deleted. Using this feature it is possible to get * a pointer to the value (for instance in order to release it), without * having to call ValkeyModule_DictGet() before deleting the key. */ int VM_DictDelC(ValkeyModuleDict *d, void *key, size_t keylen, void *oldval) { int retval = raxRemove(d->rax, key, keylen, oldval); return retval ? VALKEYMODULE_OK : VALKEYMODULE_ERR; } /* Like ValkeyModule_DictDelC() but gets the key as a ValkeyModuleString. */ int VM_DictDel(ValkeyModuleDict *d, ValkeyModuleString *key, void *oldval) { return VM_DictDelC(d, key->ptr, sdslen(key->ptr), oldval); } /* Return an iterator, setup in order to start iterating from the specified * key by applying the operator 'op', which is just a string specifying the * comparison operator to use in order to seek the first element. The * operators available are: * * * `^` -- Seek the first (lexicographically smaller) key. * * `$` -- Seek the last (lexicographically bigger) key. * * `>` -- Seek the first element greater than the specified key. * * `>=` -- Seek the first element greater or equal than the specified key. * * `<` -- Seek the first element smaller than the specified key. * * `<=` -- Seek the first element smaller or equal than the specified key. * * `==` -- Seek the first element matching exactly the specified key. * * Note that for `^` and `$` the passed key is not used, and the user may * just pass NULL with a length of 0. * * If the element to start the iteration cannot be seeked based on the * key and operator passed, ValkeyModule_DictNext() / Prev() will just return * VALKEYMODULE_ERR at the first call, otherwise they'll produce elements. */ ValkeyModuleDictIter *VM_DictIteratorStartC(ValkeyModuleDict *d, const char *op, void *key, size_t keylen) { ValkeyModuleDictIter *di = zmalloc(sizeof(*di)); di->dict = d; raxStart(&di->ri, d->rax); raxSeek(&di->ri, op, key, keylen); return di; } /* Exactly like ValkeyModule_DictIteratorStartC, but the key is passed as a * ValkeyModuleString. */ ValkeyModuleDictIter *VM_DictIteratorStart(ValkeyModuleDict *d, const char *op, ValkeyModuleString *key) { return VM_DictIteratorStartC(d, op, key->ptr, sdslen(key->ptr)); } /* Release the iterator created with ValkeyModule_DictIteratorStart(). This call * is mandatory otherwise a memory leak is introduced in the module. */ void VM_DictIteratorStop(ValkeyModuleDictIter *di) { raxStop(&di->ri); zfree(di); } /* After its creation with ValkeyModule_DictIteratorStart(), it is possible to * change the currently selected element of the iterator by using this * API call. The result based on the operator and key is exactly like * the function ValkeyModule_DictIteratorStart(), however in this case the * return value is just VALKEYMODULE_OK in case the seeked element was found, * or VALKEYMODULE_ERR in case it was not possible to seek the specified * element. It is possible to reseek an iterator as many times as you want. */ int VM_DictIteratorReseekC(ValkeyModuleDictIter *di, const char *op, void *key, size_t keylen) { return raxSeek(&di->ri, op, key, keylen); } /* Like ValkeyModule_DictIteratorReseekC() but takes the key as a * ValkeyModuleString. */ int VM_DictIteratorReseek(ValkeyModuleDictIter *di, const char *op, ValkeyModuleString *key) { return VM_DictIteratorReseekC(di, op, key->ptr, sdslen(key->ptr)); } /* Return the current item of the dictionary iterator `di` and steps to the * next element. If the iterator already yield the last element and there * are no other elements to return, NULL is returned, otherwise a pointer * to a string representing the key is provided, and the `*keylen` length * is set by reference (if keylen is not NULL). The `*dataptr`, if not NULL * is set to the value of the pointer stored at the returned key as auxiliary * data (as set by the ValkeyModule_DictSet API). * * Usage example: * * ... create the iterator here ... * char *key; * void *data; * while((key = ValkeyModule_DictNextC(iter,&keylen,&data)) != NULL) { * printf("%.*s %p\n", (int)keylen, key, data); * } * * The returned pointer is of type void because sometimes it makes sense * to cast it to a `char*` sometimes to an unsigned `char*` depending on the * fact it contains or not binary data, so this API ends being more * comfortable to use. * * The validity of the returned pointer is until the next call to the * next/prev iterator step. Also the pointer is no longer valid once the * iterator is released. */ void *VM_DictNextC(ValkeyModuleDictIter *di, size_t *keylen, void **dataptr) { if (!raxNext(&di->ri)) return NULL; if (keylen) *keylen = di->ri.key_len; if (dataptr) *dataptr = di->ri.data; return di->ri.key; } /* This function is exactly like ValkeyModule_DictNext() but after returning * the currently selected element in the iterator, it selects the previous * element (lexicographically smaller) instead of the next one. */ void *VM_DictPrevC(ValkeyModuleDictIter *di, size_t *keylen, void **dataptr) { if (!raxPrev(&di->ri)) return NULL; if (keylen) *keylen = di->ri.key_len; if (dataptr) *dataptr = di->ri.data; return di->ri.key; } /* Like ValkeyModuleNextC(), but instead of returning an internally allocated * buffer and key length, it returns directly a module string object allocated * in the specified context 'ctx' (that may be NULL exactly like for the main * API ValkeyModule_CreateString). * * The returned string object should be deallocated after use, either manually * or by using a context that has automatic memory management active. */ ValkeyModuleString *VM_DictNext(ValkeyModuleCtx *ctx, ValkeyModuleDictIter *di, void **dataptr) { size_t keylen; void *key = VM_DictNextC(di, &keylen, dataptr); if (key == NULL) return NULL; return VM_CreateString(ctx, key, keylen); } /* Like ValkeyModule_DictNext() but after returning the currently selected * element in the iterator, it selects the previous element (lexicographically * smaller) instead of the next one. */ ValkeyModuleString *VM_DictPrev(ValkeyModuleCtx *ctx, ValkeyModuleDictIter *di, void **dataptr) { size_t keylen; void *key = VM_DictPrevC(di, &keylen, dataptr); if (key == NULL) return NULL; return VM_CreateString(ctx, key, keylen); } /* Compare the element currently pointed by the iterator to the specified * element given by key/keylen, according to the operator 'op' (the set of * valid operators are the same valid for ValkeyModule_DictIteratorStart). * If the comparison is successful the command returns VALKEYMODULE_OK * otherwise VALKEYMODULE_ERR is returned. * * This is useful when we want to just emit a lexicographical range, so * in the loop, as we iterate elements, we can also check if we are still * on range. * * The function return VALKEYMODULE_ERR if the iterator reached the * end of elements condition as well. */ int VM_DictCompareC(ValkeyModuleDictIter *di, const char *op, void *key, size_t keylen) { if (raxEOF(&di->ri)) return VALKEYMODULE_ERR; int res = raxCompare(&di->ri, op, key, keylen); return res ? VALKEYMODULE_OK : VALKEYMODULE_ERR; } /* Like ValkeyModule_DictCompareC but gets the key to compare with the current * iterator key as a ValkeyModuleString. */ int VM_DictCompare(ValkeyModuleDictIter *di, const char *op, ValkeyModuleString *key) { if (raxEOF(&di->ri)) return VALKEYMODULE_ERR; int res = raxCompare(&di->ri, op, key->ptr, sdslen(key->ptr)); return res ? VALKEYMODULE_OK : VALKEYMODULE_ERR; } /* -------------------------------------------------------------------------- * ## Modules Info fields * -------------------------------------------------------------------------- */ int VM_InfoEndDictField(ValkeyModuleInfoCtx *ctx); /* Used to start a new section, before adding any fields. the section name will * be prefixed by `_` and must only include A-Z,a-z,0-9. * NULL or empty string indicates the default section (only ``) is used. * When return value is VALKEYMODULE_ERR, the section should and will be skipped. */ int VM_InfoAddSection(ValkeyModuleInfoCtx *ctx, const char *name) { sds full_name = sdsdup(ctx->module->name); if (name != NULL && strlen(name) > 0) full_name = sdscatfmt(full_name, "_%s", name); /* Implicitly end dicts, instead of returning an error which is likely un checked. */ if (ctx->in_dict_field) VM_InfoEndDictField(ctx); /* proceed only if: * 1) no section was requested (emit all) * 2) the module name was requested (emit all) * 3) this specific section was requested. */ if (ctx->requested_sections) { if ((!full_name || !dictFind(ctx->requested_sections, full_name)) && (!dictFind(ctx->requested_sections, ctx->module->name))) { sdsfree(full_name); ctx->in_section = 0; return VALKEYMODULE_ERR; } } if (ctx->sections++) ctx->info = sdscat(ctx->info, "\r\n"); ctx->info = sdscatfmt(ctx->info, "# %S\r\n", full_name); ctx->in_section = 1; sdsfree(full_name); return VALKEYMODULE_OK; } /* Starts a dict field, similar to the ones in INFO KEYSPACE. Use normal * ValkeyModule_InfoAddField* functions to add the items to this field, and * terminate with ValkeyModule_InfoEndDictField. */ int VM_InfoBeginDictField(ValkeyModuleInfoCtx *ctx, const char *name) { if (!ctx->in_section) return VALKEYMODULE_ERR; /* Implicitly end dicts, instead of returning an error which is likely un checked. */ if (ctx->in_dict_field) VM_InfoEndDictField(ctx); char *tmpmodname, *tmpname; ctx->info = sdscatfmt(ctx->info, "%s_%s:", getSafeInfoString(ctx->module->name, strlen(ctx->module->name), &tmpmodname), getSafeInfoString(name, strlen(name), &tmpname)); if (tmpmodname != NULL) zfree(tmpmodname); if (tmpname != NULL) zfree(tmpname); ctx->in_dict_field = 1; return VALKEYMODULE_OK; } /* Ends a dict field, see ValkeyModule_InfoBeginDictField */ int VM_InfoEndDictField(ValkeyModuleInfoCtx *ctx) { if (!ctx->in_dict_field) return VALKEYMODULE_ERR; /* trim the last ',' if found. */ if (ctx->info[sdslen(ctx->info) - 1] == ',') sdsIncrLen(ctx->info, -1); ctx->info = sdscat(ctx->info, "\r\n"); ctx->in_dict_field = 0; return VALKEYMODULE_OK; } /* Used by ValkeyModuleInfoFunc to add info fields. * Each field will be automatically prefixed by `_`. * Field names or values must not include `\r\n` or `:`. */ int VM_InfoAddFieldString(ValkeyModuleInfoCtx *ctx, const char *field, ValkeyModuleString *value) { if (!ctx->in_section) return VALKEYMODULE_ERR; if (ctx->in_dict_field) { ctx->info = sdscatfmt(ctx->info, "%s=%S,", field, (sds)value->ptr); return VALKEYMODULE_OK; } ctx->info = sdscatfmt(ctx->info, "%s_%s:%S\r\n", ctx->module->name, field, (sds)value->ptr); return VALKEYMODULE_OK; } /* See ValkeyModule_InfoAddFieldString(). */ int VM_InfoAddFieldCString(ValkeyModuleInfoCtx *ctx, const char *field, const char *value) { if (!ctx->in_section) return VALKEYMODULE_ERR; if (ctx->in_dict_field) { ctx->info = sdscatfmt(ctx->info, "%s=%s,", field, value); return VALKEYMODULE_OK; } ctx->info = sdscatfmt(ctx->info, "%s_%s:%s\r\n", ctx->module->name, field, value); return VALKEYMODULE_OK; } /* See ValkeyModule_InfoAddFieldString(). */ int VM_InfoAddFieldDouble(ValkeyModuleInfoCtx *ctx, const char *field, double value) { if (!ctx->in_section) return VALKEYMODULE_ERR; if (ctx->in_dict_field) { ctx->info = sdscatprintf(ctx->info, "%s=%.17g,", field, value); return VALKEYMODULE_OK; } ctx->info = sdscatprintf(ctx->info, "%s_%s:%.17g\r\n", ctx->module->name, field, value); return VALKEYMODULE_OK; } /* See ValkeyModule_InfoAddFieldString(). */ int VM_InfoAddFieldLongLong(ValkeyModuleInfoCtx *ctx, const char *field, long long value) { if (!ctx->in_section) return VALKEYMODULE_ERR; if (ctx->in_dict_field) { ctx->info = sdscatfmt(ctx->info, "%s=%I,", field, value); return VALKEYMODULE_OK; } ctx->info = sdscatfmt(ctx->info, "%s_%s:%I\r\n", ctx->module->name, field, value); return VALKEYMODULE_OK; } /* See ValkeyModule_InfoAddFieldString(). */ int VM_InfoAddFieldULongLong(ValkeyModuleInfoCtx *ctx, const char *field, unsigned long long value) { if (!ctx->in_section) return VALKEYMODULE_ERR; if (ctx->in_dict_field) { ctx->info = sdscatfmt(ctx->info, "%s=%U,", field, value); return VALKEYMODULE_OK; } ctx->info = sdscatfmt(ctx->info, "%s_%s:%U\r\n", ctx->module->name, field, value); return VALKEYMODULE_OK; } /* Registers callback for the INFO command. The callback should add INFO fields * by calling the `ValkeyModule_InfoAddField*()` functions. */ int VM_RegisterInfoFunc(ValkeyModuleCtx *ctx, ValkeyModuleInfoFunc cb) { ctx->module->info_cb = cb; return VALKEYMODULE_OK; } sds modulesCollectInfo(sds info, dict *sections_dict, int for_crash_report, int sections) { dictIterator *di = dictGetIterator(modules); dictEntry *de; while ((de = dictNext(di)) != NULL) { struct ValkeyModule *module = dictGetVal(de); if (!module->info_cb) continue; ValkeyModuleInfoCtx info_ctx = {module, sections_dict, info, sections, 0, 0}; module->info_cb(&info_ctx, for_crash_report); /* Implicitly end dicts (no way to handle errors, and we must add the newline). */ if (info_ctx.in_dict_field) VM_InfoEndDictField(&info_ctx); info = info_ctx.info; sections = info_ctx.sections; } dictReleaseIterator(di); return info; } /* Get information about the server similar to the one that returns from the * INFO command. This function takes an optional 'section' argument that may * be NULL. The return value holds the output and can be used with * ValkeyModule_ServerInfoGetField and alike to get the individual fields. * When done, it needs to be freed with ValkeyModule_FreeServerInfo or with the * automatic memory management mechanism if enabled. */ ValkeyModuleServerInfoData *VM_GetServerInfo(ValkeyModuleCtx *ctx, const char *section) { struct ValkeyModuleServerInfoData *d = zmalloc(sizeof(*d)); d->rax = raxNew(); if (ctx != NULL) autoMemoryAdd(ctx, VALKEYMODULE_AM_INFO, d); int all = 0, everything = 0; robj *argv[1]; argv[0] = section ? createStringObject(section, strlen(section)) : NULL; dict *section_dict = genInfoSectionDict(argv, section ? 1 : 0, NULL, &all, &everything); sds info = genValkeyInfoString(section_dict, all, everything); int totlines, i; sds *lines = sdssplitlen(info, sdslen(info), "\r\n", 2, &totlines); for (i = 0; i < totlines; i++) { sds line = lines[i]; if (line[0] == '#') continue; char *sep = strchr(line, ':'); if (!sep) continue; unsigned char *key = (unsigned char *)line; size_t keylen = (intptr_t)sep - (intptr_t)line; sds val = sdsnewlen(sep + 1, sdslen(line) - ((intptr_t)sep - (intptr_t)line) - 1); if (!raxTryInsert(d->rax, key, keylen, val, NULL)) sdsfree(val); } sdsfree(info); sdsfreesplitres(lines, totlines); releaseInfoSectionDict(section_dict); if (argv[0]) decrRefCount(argv[0]); return d; } /* Free data created with VM_GetServerInfo(). You need to pass the * context pointer 'ctx' only if the dictionary was created using the * context instead of passing NULL. */ void VM_FreeServerInfo(ValkeyModuleCtx *ctx, ValkeyModuleServerInfoData *data) { if (ctx != NULL) autoMemoryFreed(ctx, VALKEYMODULE_AM_INFO, data); raxFreeWithCallback(data->rax, sdsfreeVoid); zfree(data); } /* Get the value of a field from data collected with VM_GetServerInfo(). You * need to pass the context pointer 'ctx' only if you want to use auto memory * mechanism to release the returned string. Return value will be NULL if the * field was not found. */ ValkeyModuleString *VM_ServerInfoGetField(ValkeyModuleCtx *ctx, ValkeyModuleServerInfoData *data, const char *field) { void *result; if (!raxFind(data->rax, (unsigned char *)field, strlen(field), &result)) return NULL; sds val = result; ValkeyModuleString *o = createStringObject(val, sdslen(val)); if (ctx != NULL) autoMemoryAdd(ctx, VALKEYMODULE_AM_STRING, o); return o; } /* Similar to VM_ServerInfoGetField, but returns a char* which should not be freed but the caller. */ const char *VM_ServerInfoGetFieldC(ValkeyModuleServerInfoData *data, const char *field) { void *result = NULL; raxFind(data->rax, (unsigned char *)field, strlen(field), &result); return result; } /* Get the value of a field from data collected with VM_GetServerInfo(). If the * field is not found, or is not numerical or out of range, return value will be * 0, and the optional out_err argument will be set to VALKEYMODULE_ERR. */ long long VM_ServerInfoGetFieldSigned(ValkeyModuleServerInfoData *data, const char *field, int *out_err) { long long ll; void *result; if (!raxFind(data->rax, (unsigned char *)field, strlen(field), &result)) { if (out_err) *out_err = VALKEYMODULE_ERR; return 0; } sds val = result; if (!string2ll(val, sdslen(val), &ll)) { if (out_err) *out_err = VALKEYMODULE_ERR; return 0; } if (out_err) *out_err = VALKEYMODULE_OK; return ll; } /* Get the value of a field from data collected with VM_GetServerInfo(). If the * field is not found, or is not numerical or out of range, return value will be * 0, and the optional out_err argument will be set to VALKEYMODULE_ERR. */ unsigned long long VM_ServerInfoGetFieldUnsigned(ValkeyModuleServerInfoData *data, const char *field, int *out_err) { unsigned long long ll; void *result; if (!raxFind(data->rax, (unsigned char *)field, strlen(field), &result)) { if (out_err) *out_err = VALKEYMODULE_ERR; return 0; } sds val = result; if (!string2ull(val, &ll)) { if (out_err) *out_err = VALKEYMODULE_ERR; return 0; } if (out_err) *out_err = VALKEYMODULE_OK; return ll; } /* Get the value of a field from data collected with VM_GetServerInfo(). If the * field is not found, or is not a double, return value will be 0, and the * optional out_err argument will be set to VALKEYMODULE_ERR. */ double VM_ServerInfoGetFieldDouble(ValkeyModuleServerInfoData *data, const char *field, int *out_err) { double dbl; void *result; if (!raxFind(data->rax, (unsigned char *)field, strlen(field), &result)) { if (out_err) *out_err = VALKEYMODULE_ERR; return 0; } sds val = result; if (!string2d(val, sdslen(val), &dbl)) { if (out_err) *out_err = VALKEYMODULE_ERR; return 0; } if (out_err) *out_err = VALKEYMODULE_OK; return dbl; } /* -------------------------------------------------------------------------- * ## Modules utility APIs * -------------------------------------------------------------------------- */ /* Return random bytes using SHA1 in counter mode with a /dev/urandom * initialized seed. This function is fast so can be used to generate * many bytes without any effect on the operating system entropy pool. * Currently this function is not thread safe. */ void VM_GetRandomBytes(unsigned char *dst, size_t len) { getRandomBytes(dst, len); } /* Like ValkeyModule_GetRandomBytes() but instead of setting the string to * random bytes the string is set to random characters in the in the * hex charset [0-9a-f]. */ void VM_GetRandomHexChars(char *dst, size_t len) { getRandomHexChars(dst, len); } /* -------------------------------------------------------------------------- * ## Modules API exporting / importing * -------------------------------------------------------------------------- */ /* This function is called by a module in order to export some API with a * given name. Other modules will be able to use this API by calling the * symmetrical function VM_GetSharedAPI() and casting the return value to * the right function pointer. * * The function will return VALKEYMODULE_OK if the name is not already taken, * otherwise VALKEYMODULE_ERR will be returned and no operation will be * performed. * * IMPORTANT: the apiname argument should be a string literal with static * lifetime. The API relies on the fact that it will always be valid in * the future. */ int VM_ExportSharedAPI(ValkeyModuleCtx *ctx, const char *apiname, void *func) { ValkeyModuleSharedAPI *sapi = zmalloc(sizeof(*sapi)); sapi->module = ctx->module; sapi->func = func; if (dictAdd(server.sharedapi, (char *)apiname, sapi) != DICT_OK) { zfree(sapi); return VALKEYMODULE_ERR; } return VALKEYMODULE_OK; } /* Request an exported API pointer. The return value is just a void pointer * that the caller of this function will be required to cast to the right * function pointer, so this is a private contract between modules. * * If the requested API is not available then NULL is returned. Because * modules can be loaded at different times with different order, this * function calls should be put inside some module generic API registering * step, that is called every time a module attempts to execute a * command that requires external APIs: if some API cannot be resolved, the * command should return an error. * * Here is an example: * * int ... myCommandImplementation(void) { * if (getExternalAPIs() == 0) { * reply with an error here if we cannot have the APIs * } * // Use the API: * myFunctionPointer(foo); * } * * And the function registerAPI() is: * * int getExternalAPIs(void) { * static int api_loaded = 0; * if (api_loaded != 0) return 1; // APIs already resolved. * * myFunctionPointer = ValkeyModule_GetSharedAPI("..."); * if (myFunctionPointer == NULL) return 0; * * return 1; * } */ void *VM_GetSharedAPI(ValkeyModuleCtx *ctx, const char *apiname) { dictEntry *de = dictFind(server.sharedapi, apiname); if (de == NULL) return NULL; ValkeyModuleSharedAPI *sapi = dictGetVal(de); if (listSearchKey(sapi->module->usedby, ctx->module) == NULL) { listAddNodeTail(sapi->module->usedby, ctx->module); listAddNodeTail(ctx->module->using, sapi->module); } return sapi->func; } /* Remove all the APIs registered by the specified module. Usually you * want this when the module is going to be unloaded. This function * assumes that's caller responsibility to make sure the APIs are not * used by other modules. * * The number of unregistered APIs is returned. */ int moduleUnregisterSharedAPI(ValkeyModule *module) { int count = 0; dictIterator *di = dictGetSafeIterator(server.sharedapi); dictEntry *de; while ((de = dictNext(di)) != NULL) { const char *apiname = dictGetKey(de); ValkeyModuleSharedAPI *sapi = dictGetVal(de); if (sapi->module == module) { dictDelete(server.sharedapi, apiname); zfree(sapi); count++; } } dictReleaseIterator(di); return count; } /* Remove the specified module as an user of APIs of ever other module. * This is usually called when a module is unloaded. * * Returns the number of modules this module was using APIs from. */ int moduleUnregisterUsedAPI(ValkeyModule *module) { listIter li; listNode *ln; int count = 0; listRewind(module->using, &li); while ((ln = listNext(&li))) { ValkeyModule *used = ln->value; listNode *ln = listSearchKey(used->usedby, module); if (ln) { listDelNode(used->usedby, ln); count++; } } return count; } /* Unregister all filters registered by a module. * This is called when a module is being unloaded. * * Returns the number of filters unregistered. */ int moduleUnregisterFilters(ValkeyModule *module) { listIter li; listNode *ln; int count = 0; listRewind(module->filters, &li); while ((ln = listNext(&li))) { ValkeyModuleCommandFilter *filter = ln->value; listNode *ln = listSearchKey(moduleCommandFilters, filter); if (ln) { listDelNode(moduleCommandFilters, ln); count++; } zfree(filter); } return count; } /* -------------------------------------------------------------------------- * ## Module Command Filter API * -------------------------------------------------------------------------- */ /* Register a new command filter function. * * Command filtering makes it possible for modules to extend the server by plugging * into the execution flow of all commands. * * A registered filter gets called before the server executes *any* command. This * includes both core server commands and commands registered by any module. The * filter applies in all execution paths including: * * 1. Invocation by a client. * 2. Invocation through `ValkeyModule_Call()` by any module. * 3. Invocation through Lua `redis.call()`. * 4. Replication of a command from a primary. * * The filter executes in a special filter context, which is different and more * limited than a ValkeyModuleCtx. Because the filter affects any command, it * must be implemented in a very efficient way to reduce the performance impact * on the server. All Module API calls that require a valid context (such as * `ValkeyModule_Call()`, `ValkeyModule_OpenKey()`, etc.) are not supported in a * filter context. * * The `ValkeyModuleCommandFilterCtx` can be used to inspect or modify the * executed command and its arguments. As the filter executes before the server * begins processing the command, any change will affect the way the command is * processed. For example, a module can override server commands this way: * * 1. Register a `MODULE.SET` command which implements an extended version of * the `SET` command. * 2. Register a command filter which detects invocation of `SET` on a specific * pattern of keys. Once detected, the filter will replace the first * argument from `SET` to `MODULE.SET`. * 3. When filter execution is complete, the server considers the new command name * and therefore executes the module's own command. * * Note that in the above use case, if `MODULE.SET` itself uses * `ValkeyModule_Call()` the filter will be applied on that call as well. If * that is not desired, the `VALKEYMODULE_CMDFILTER_NOSELF` flag can be set when * registering the filter. * * The `VALKEYMODULE_CMDFILTER_NOSELF` flag prevents execution flows that * originate from the module's own `VM_Call()` from reaching the filter. This * flag is effective for all execution flows, including nested ones, as long as * the execution begins from the module's command context or a thread-safe * context that is associated with a blocking command. * * Detached thread-safe contexts are *not* associated with the module and cannot * be protected by this flag. * * If multiple filters are registered (by the same or different modules), they * are executed in the order of registration. */ ValkeyModuleCommandFilter * VM_RegisterCommandFilter(ValkeyModuleCtx *ctx, ValkeyModuleCommandFilterFunc callback, int flags) { ValkeyModuleCommandFilter *filter = zmalloc(sizeof(*filter)); filter->module = ctx->module; filter->callback = callback; filter->flags = flags; listAddNodeTail(moduleCommandFilters, filter); listAddNodeTail(ctx->module->filters, filter); return filter; } /* Unregister a command filter. */ int VM_UnregisterCommandFilter(ValkeyModuleCtx *ctx, ValkeyModuleCommandFilter *filter) { listNode *ln; /* A module can only remove its own filters */ if (filter->module != ctx->module) return VALKEYMODULE_ERR; ln = listSearchKey(moduleCommandFilters, filter); if (!ln) return VALKEYMODULE_ERR; listDelNode(moduleCommandFilters, ln); ln = listSearchKey(ctx->module->filters, filter); if (!ln) return VALKEYMODULE_ERR; /* Shouldn't happen */ listDelNode(ctx->module->filters, ln); zfree(filter); return VALKEYMODULE_OK; } void moduleCallCommandFilters(client *c) { if (listLength(moduleCommandFilters) == 0) return; listIter li; listNode *ln; listRewind(moduleCommandFilters, &li); ValkeyModuleCommandFilterCtx filter = {.argv = c->argv, .argv_len = c->argv_len, .argc = c->argc, .c = c}; robj *tmp = c->argv[0]; incrRefCount(tmp); while ((ln = listNext(&li))) { ValkeyModuleCommandFilter *f = ln->value; /* Skip filter if VALKEYMODULE_CMDFILTER_NOSELF is set and module is * currently processing a command. */ if ((f->flags & VALKEYMODULE_CMDFILTER_NOSELF) && f->module->in_call) continue; /* Call filter */ f->callback(&filter); } c->argv = filter.argv; c->argv_len = filter.argv_len; c->argc = filter.argc; if (tmp != c->argv[0]) { /* With I/O thread command-lookup offload, we set c->io_parsed_cmd to the command corresponding to c->argv[0]. * Since the command filter just changed it, we need to reset c->io_parsed_cmd to null. */ c->io_parsed_cmd = NULL; } decrRefCount(tmp); } /* Return the number of arguments a filtered command has. The number of * arguments include the command itself. */ int VM_CommandFilterArgsCount(ValkeyModuleCommandFilterCtx *fctx) { return fctx->argc; } /* Return the specified command argument. The first argument (position 0) is * the command itself, and the rest are user-provided args. */ ValkeyModuleString *VM_CommandFilterArgGet(ValkeyModuleCommandFilterCtx *fctx, int pos) { if (pos < 0 || pos >= fctx->argc) return NULL; return fctx->argv[pos]; } /* Modify the filtered command by inserting a new argument at the specified * position. The specified ValkeyModuleString argument may be used by the server * after the filter context is destroyed, so it must not be auto-memory * allocated, freed or used elsewhere. */ int VM_CommandFilterArgInsert(ValkeyModuleCommandFilterCtx *fctx, int pos, ValkeyModuleString *arg) { int i; if (pos < 0 || pos > fctx->argc) return VALKEYMODULE_ERR; if (fctx->argv_len < fctx->argc + 1) { fctx->argv_len = fctx->argc + 1; fctx->argv = zrealloc(fctx->argv, fctx->argv_len * sizeof(ValkeyModuleString *)); } for (i = fctx->argc; i > pos; i--) { fctx->argv[i] = fctx->argv[i - 1]; } fctx->argv[pos] = arg; fctx->argc++; return VALKEYMODULE_OK; } /* Modify the filtered command by replacing an existing argument with a new one. * The specified ValkeyModuleString argument may be used by the server after the * filter context is destroyed, so it must not be auto-memory allocated, freed * or used elsewhere. */ int VM_CommandFilterArgReplace(ValkeyModuleCommandFilterCtx *fctx, int pos, ValkeyModuleString *arg) { if (pos < 0 || pos >= fctx->argc) return VALKEYMODULE_ERR; decrRefCount(fctx->argv[pos]); fctx->argv[pos] = arg; return VALKEYMODULE_OK; } /* Modify the filtered command by deleting an argument at the specified * position. */ int VM_CommandFilterArgDelete(ValkeyModuleCommandFilterCtx *fctx, int pos) { int i; if (pos < 0 || pos >= fctx->argc) return VALKEYMODULE_ERR; decrRefCount(fctx->argv[pos]); for (i = pos; i < fctx->argc - 1; i++) { fctx->argv[i] = fctx->argv[i + 1]; } fctx->argc--; return VALKEYMODULE_OK; } /* Get Client ID for client that issued the command we are filtering */ unsigned long long VM_CommandFilterGetClientId(ValkeyModuleCommandFilterCtx *fctx) { return fctx->c->id; } /* For a given pointer allocated via ValkeyModule_Alloc() or * ValkeyModule_Realloc(), return the amount of memory allocated for it. * Note that this may be different (larger) than the memory we allocated * with the allocation calls, since sometimes the underlying allocator * will allocate more memory. */ size_t VM_MallocSize(void *ptr) { return zmalloc_size(ptr); } /* Similar to VM_MallocSize, the difference is that VM_MallocUsableSize * returns the usable size of memory by the module. */ size_t VM_MallocUsableSize(void *ptr) { /* It is safe to use 'zmalloc_usable_size()' to manipulate additional * memory space, as we guarantee that the compiler can recognize this * after 'VM_Alloc', 'VM_TryAlloc', 'VM_Realloc', or 'VM_Calloc'. */ return zmalloc_usable_size(ptr); } /* Same as VM_MallocSize, except it works on ValkeyModuleString pointers. */ size_t VM_MallocSizeString(ValkeyModuleString *str) { serverAssert(str->type == OBJ_STRING); return sizeof(*str) + getStringObjectSdsUsedMemory(str); } /* Same as VM_MallocSize, except it works on ValkeyModuleDict pointers. * Note that the returned value is only the overhead of the underlying structures, * it does not include the allocation size of the keys and values. */ size_t VM_MallocSizeDict(ValkeyModuleDict *dict) { size_t size = sizeof(ValkeyModuleDict); size += raxAllocSize(dict->rax); return size; } /* Return the a number between 0 to 1 indicating the amount of memory * currently used, relative to the server "maxmemory" configuration. * * * 0 - No memory limit configured. * * Between 0 and 1 - The percentage of the memory used normalized in 0-1 range. * * Exactly 1 - Memory limit reached. * * Greater 1 - More memory used than the configured limit. */ float VM_GetUsedMemoryRatio(void) { float level; getMaxmemoryState(NULL, NULL, NULL, &level); return level; } /* -------------------------------------------------------------------------- * ## Scanning keyspace and hashes * -------------------------------------------------------------------------- */ typedef void (*ValkeyModuleScanCB)(ValkeyModuleCtx *ctx, ValkeyModuleString *keyname, ValkeyModuleKey *key, void *privdata); typedef struct { ValkeyModuleCtx *ctx; void *user_data; ValkeyModuleScanCB fn; } ScanCBData; typedef struct ValkeyModuleScanCursor { unsigned long long cursor; int done; } ValkeyModuleScanCursor; static void moduleScanCallback(void *privdata, void *element) { ScanCBData *data = privdata; robj *val = element; sds key = objectGetKey(val); ValkeyModuleString *keyname = createObject(OBJ_STRING, sdsdup(key)); /* Setup the key handle. */ ValkeyModuleKey kp = {0}; moduleInitKey(&kp, data->ctx, keyname, val, VALKEYMODULE_READ); data->fn(data->ctx, keyname, &kp, data->user_data); moduleCloseKey(&kp); decrRefCount(keyname); } /* Create a new cursor to be used with ValkeyModule_Scan */ ValkeyModuleScanCursor *VM_ScanCursorCreate(void) { ValkeyModuleScanCursor *cursor = zmalloc(sizeof(*cursor)); cursor->cursor = 0; cursor->done = 0; return cursor; } /* Restart an existing cursor. The keys will be rescanned. */ void VM_ScanCursorRestart(ValkeyModuleScanCursor *cursor) { cursor->cursor = 0; cursor->done = 0; } /* Destroy the cursor struct. */ void VM_ScanCursorDestroy(ValkeyModuleScanCursor *cursor) { zfree(cursor); } /* Scan API that allows a module to scan all the keys and value in * the selected db. * * Callback for scan implementation. * * void scan_callback(ValkeyModuleCtx *ctx, ValkeyModuleString *keyname, * ValkeyModuleKey *key, void *privdata); * * - `ctx`: the module context provided to for the scan. * - `keyname`: owned by the caller and need to be retained if used after this * function. * - `key`: holds info on the key and value, it is provided as best effort, in * some cases it might be NULL, in which case the user should (can) use * ValkeyModule_OpenKey() (and CloseKey too). * when it is provided, it is owned by the caller and will be free when the * callback returns. * - `privdata`: the user data provided to ValkeyModule_Scan(). * * The way it should be used: * * ValkeyModuleScanCursor *c = ValkeyModule_ScanCursorCreate(); * while(ValkeyModule_Scan(ctx, c, callback, privateData)); * ValkeyModule_ScanCursorDestroy(c); * * It is also possible to use this API from another thread while the lock * is acquired during the actual call to VM_Scan: * * ValkeyModuleScanCursor *c = ValkeyModule_ScanCursorCreate(); * ValkeyModule_ThreadSafeContextLock(ctx); * while(ValkeyModule_Scan(ctx, c, callback, privateData)){ * ValkeyModule_ThreadSafeContextUnlock(ctx); * // do some background job * ValkeyModule_ThreadSafeContextLock(ctx); * } * ValkeyModule_ScanCursorDestroy(c); * * The function will return 1 if there are more elements to scan and * 0 otherwise, possibly setting errno if the call failed. * * It is also possible to restart an existing cursor using VM_ScanCursorRestart. * * IMPORTANT: This API is very similar to the SCAN command from the * point of view of the guarantees it provides. This means that the API * may report duplicated keys, but guarantees to report at least one time * every key that was there from the start to the end of the scanning process. * * NOTE: If you do database changes within the callback, you should be aware * that the internal state of the database may change. For instance it is safe * to delete or modify the current key, but may not be safe to delete any * other key. * Moreover playing with the keyspace while iterating may have the * effect of returning more duplicates. A safe pattern is to store the keys * names you want to modify elsewhere, and perform the actions on the keys * later when the iteration is complete. However this can cost a lot of * memory, so it may make sense to just operate on the current key when * possible during the iteration, given that this is safe. */ int VM_Scan(ValkeyModuleCtx *ctx, ValkeyModuleScanCursor *cursor, ValkeyModuleScanCB fn, void *privdata) { if (cursor->done) { errno = ENOENT; return 0; } int ret = 1; ScanCBData data = {ctx, privdata, fn}; cursor->cursor = dbScan(ctx->client->db, cursor->cursor, moduleScanCallback, &data); if (cursor->cursor == 0) { cursor->done = 1; ret = 0; } errno = 0; return ret; } typedef void (*ValkeyModuleScanKeyCB)(ValkeyModuleKey *key, ValkeyModuleString *field, ValkeyModuleString *value, void *privdata); typedef struct { ValkeyModuleKey *key; void *user_data; ValkeyModuleScanKeyCB fn; } ScanKeyCBData; static void moduleScanKeyHashtableCallback(void *privdata, void *entry) { ScanKeyCBData *data = privdata; robj *o = data->key->value; robj *value = NULL; sds key = NULL; if (o->type == OBJ_SET) { key = entry; /* no value */ } else if (o->type == OBJ_ZSET) { zskiplistNode *node = (zskiplistNode *)entry; key = node->ele; value = createStringObjectFromLongDouble(node->score, 0); } else if (o->type == OBJ_HASH) { key = hashTypeEntryGetField(entry); sds val = hashTypeEntryGetValue(entry); value = createStringObject(val, sdslen(val)); } else { serverPanic("unexpected object type"); } robj *field = createStringObject(key, sdslen(key)); data->fn(data->key, field, value, data->user_data); decrRefCount(field); if (value) decrRefCount(value); } /* Scan api that allows a module to scan the elements in a hash, set or sorted set key * * Callback for scan implementation. * * void scan_callback(ValkeyModuleKey *key, ValkeyModuleString* field, ValkeyModuleString* value, void *privdata); * * - key - the key context provided to for the scan. * - field - field name, owned by the caller and need to be retained if used * after this function. * - value - value string or NULL for set type, owned by the caller and need to * be retained if used after this function. * - privdata - the user data provided to ValkeyModule_ScanKey. * * The way it should be used: * * ValkeyModuleScanCursor *c = ValkeyModule_ScanCursorCreate(); * ValkeyModuleKey *key = ValkeyModule_OpenKey(...) * while(ValkeyModule_ScanKey(key, c, callback, privateData)); * ValkeyModule_CloseKey(key); * ValkeyModule_ScanCursorDestroy(c); * * It is also possible to use this API from another thread while the lock is acquired during * the actual call to VM_ScanKey, and re-opening the key each time: * * ValkeyModuleScanCursor *c = ValkeyModule_ScanCursorCreate(); * ValkeyModule_ThreadSafeContextLock(ctx); * ValkeyModuleKey *key = ValkeyModule_OpenKey(...) * while(ValkeyModule_ScanKey(ctx, c, callback, privateData)){ * ValkeyModule_CloseKey(key); * ValkeyModule_ThreadSafeContextUnlock(ctx); * // do some background job * ValkeyModule_ThreadSafeContextLock(ctx); * ValkeyModuleKey *key = ValkeyModule_OpenKey(...) * } * ValkeyModule_CloseKey(key); * ValkeyModule_ScanCursorDestroy(c); * * The function will return 1 if there are more elements to scan and 0 otherwise, * possibly setting errno if the call failed. * It is also possible to restart an existing cursor using VM_ScanCursorRestart. * * NOTE: Certain operations are unsafe while iterating the object. For instance * while the API guarantees to return at least one time all the elements that * are present in the data structure consistently from the start to the end * of the iteration (see HSCAN and similar commands documentation), the more * you play with the elements, the more duplicates you may get. In general * deleting the current element of the data structure is safe, while removing * the key you are iterating is not safe. */ int VM_ScanKey(ValkeyModuleKey *key, ValkeyModuleScanCursor *cursor, ValkeyModuleScanKeyCB fn, void *privdata) { if (key == NULL || key->value == NULL) { errno = EINVAL; return 0; } hashtable *ht = NULL; robj *o = key->value; if (o->type == OBJ_SET) { if (o->encoding == OBJ_ENCODING_HASHTABLE) ht = o->ptr; } else if (o->type == OBJ_HASH) { if (o->encoding == OBJ_ENCODING_HASHTABLE) ht = o->ptr; } else if (o->type == OBJ_ZSET) { if (o->encoding == OBJ_ENCODING_SKIPLIST) ht = ((zset *)o->ptr)->ht; } else { errno = EINVAL; return 0; } if (cursor->done) { errno = ENOENT; return 0; } int ret = 1; if (ht) { ScanKeyCBData data = {key, privdata, fn}; cursor->cursor = hashtableScan(ht, cursor->cursor, moduleScanKeyHashtableCallback, &data); if (cursor->cursor == 0) { cursor->done = 1; ret = 0; } } else if (o->type == OBJ_SET) { setTypeIterator *si = setTypeInitIterator(o); sds sdsele; while ((sdsele = setTypeNextObject(si)) != NULL) { robj *field = createObject(OBJ_STRING, sdsele); fn(key, field, NULL, privdata); decrRefCount(field); } setTypeReleaseIterator(si); cursor->cursor = 1; cursor->done = 1; ret = 0; } else if (o->type == OBJ_ZSET || o->type == OBJ_HASH) { unsigned char *p = lpSeek(o->ptr, 0); unsigned char *vstr; unsigned int vlen; long long vll; while (p) { vstr = lpGetValue(p, &vlen, &vll); robj *field = (vstr != NULL) ? createStringObject((char *)vstr, vlen) : createStringObjectFromLongLongWithSds(vll); p = lpNext(o->ptr, p); vstr = lpGetValue(p, &vlen, &vll); robj *value = (vstr != NULL) ? createStringObject((char *)vstr, vlen) : createStringObjectFromLongLongWithSds(vll); fn(key, field, value, privdata); p = lpNext(o->ptr, p); decrRefCount(field); decrRefCount(value); } cursor->cursor = 1; cursor->done = 1; ret = 0; } errno = 0; return ret; } /* -------------------------------------------------------------------------- * ## Module fork API * -------------------------------------------------------------------------- */ /* Create a background child process with the current frozen snapshot of the * main process where you can do some processing in the background without * affecting / freezing the traffic and no need for threads and GIL locking. * Note that the server allows for only one concurrent fork. * When the child wants to exit, it should call ValkeyModule_ExitFromChild. * If the parent wants to kill the child it should call ValkeyModule_KillForkChild * The done handler callback will be executed on the parent process when the * child existed (but not when killed) * Return: -1 on failure, on success the parent process will get a positive PID * of the child, and the child process will get 0. */ int VM_Fork(ValkeyModuleForkDoneHandler cb, void *user_data) { pid_t childpid; if ((childpid = serverFork(CHILD_TYPE_MODULE)) == 0) { /* Child */ if (strstr(server.exec_argv[0], "redis-server") != NULL) { serverSetProcTitle("redis-module-fork"); } else { serverSetProcTitle("valkey-module-fork"); } } else if (childpid == -1) { serverLog(LL_WARNING, "Can't fork for module: %s", strerror(errno)); } else { /* Parent */ moduleForkInfo.done_handler = cb; moduleForkInfo.done_handler_user_data = user_data; serverLog(LL_VERBOSE, "Module fork started pid: %ld ", (long)childpid); } return childpid; } /* The module is advised to call this function from the fork child once in a while, * so that it can report progress and COW memory to the parent which will be * reported in INFO. * The `progress` argument should between 0 and 1, or -1 when not available. */ void VM_SendChildHeartbeat(double progress) { sendChildInfoGeneric(CHILD_INFO_TYPE_CURRENT_INFO, 0, progress, "Module fork"); } /* Call from the child process when you want to terminate it. * retcode will be provided to the done handler executed on the parent process. */ int VM_ExitFromChild(int retcode) { sendChildCowInfo(CHILD_INFO_TYPE_MODULE_COW_SIZE, "Module fork"); exitFromChild(retcode); return VALKEYMODULE_OK; } /* Kill the active module forked child, if there is one active and the * pid matches, and returns C_OK. Otherwise if there is no active module * child or the pid does not match, return C_ERR without doing anything. */ int TerminateModuleForkChild(int child_pid, int wait) { /* Module child should be active and pid should match. */ if (server.child_type != CHILD_TYPE_MODULE || server.child_pid != child_pid) return C_ERR; int statloc; serverLog(LL_VERBOSE, "Killing running module fork child: %ld", (long)server.child_pid); if (kill(server.child_pid, SIGUSR1) != -1 && wait) { while (waitpid(server.child_pid, &statloc, 0) != server.child_pid); } /* Reset the buffer accumulating changes while the child saves. */ resetChildState(); moduleForkInfo.done_handler = NULL; moduleForkInfo.done_handler_user_data = NULL; return C_OK; } /* Can be used to kill the forked child process from the parent process. * child_pid would be the return value of ValkeyModule_Fork. */ int VM_KillForkChild(int child_pid) { /* Kill module child, wait for child exit. */ if (TerminateModuleForkChild(child_pid, 1) == C_OK) return VALKEYMODULE_OK; else return VALKEYMODULE_ERR; } void ModuleForkDoneHandler(int exitcode, int bysignal) { serverLog(LL_NOTICE, "Module fork exited pid: %ld, retcode: %d, bysignal: %d", (long)server.child_pid, exitcode, bysignal); if (moduleForkInfo.done_handler) { moduleForkInfo.done_handler(exitcode, bysignal, moduleForkInfo.done_handler_user_data); } moduleForkInfo.done_handler = NULL; moduleForkInfo.done_handler_user_data = NULL; } /* -------------------------------------------------------------------------- * ## Server hooks implementation * -------------------------------------------------------------------------- */ /* This must be synced with VALKEYMODULE_EVENT_* * We use -1 (MAX_UINT64) to denote that this event doesn't have * a data structure associated with it. We use MAX_UINT64 on purpose, * in order to pass the check in ValkeyModule_SubscribeToServerEvent. */ static uint64_t moduleEventVersions[] = { VALKEYMODULE_REPLICATIONINFO_VERSION, /* VALKEYMODULE_EVENT_REPLICATION_ROLE_CHANGED */ -1, /* VALKEYMODULE_EVENT_PERSISTENCE */ VALKEYMODULE_FLUSHINFO_VERSION, /* VALKEYMODULE_EVENT_FLUSHDB */ -1, /* VALKEYMODULE_EVENT_LOADING */ VALKEYMODULE_CLIENTINFO_VERSION, /* VALKEYMODULE_EVENT_CLIENT_CHANGE */ -1, /* VALKEYMODULE_EVENT_SHUTDOWN */ -1, /* VALKEYMODULE_EVENT_REPLICA_CHANGE */ -1, /* VALKEYMODULE_EVENT_PRIMARY_LINK_CHANGE */ VALKEYMODULE_CRON_LOOP_VERSION, /* VALKEYMODULE_EVENT_CRON_LOOP */ VALKEYMODULE_MODULE_CHANGE_VERSION, /* VALKEYMODULE_EVENT_MODULE_CHANGE */ VALKEYMODULE_LOADING_PROGRESS_VERSION, /* VALKEYMODULE_EVENT_LOADING_PROGRESS */ VALKEYMODULE_SWAPDBINFO_VERSION, /* VALKEYMODULE_EVENT_SWAPDB */ -1, /* VALKEYMODULE_EVENT_REPL_BACKUP */ -1, /* VALKEYMODULE_EVENT_FORK_CHILD */ -1, /* VALKEYMODULE_EVENT_REPL_ASYNC_LOAD */ -1, /* VALKEYMODULE_EVENT_EVENTLOOP */ -1, /* VALKEYMODULE_EVENT_CONFIG */ VALKEYMODULE_KEYINFO_VERSION, /* VALKEYMODULE_EVENT_KEY */ }; /* Register to be notified, via a callback, when the specified server event * happens. The callback is called with the event as argument, and an additional * argument which is a void pointer and should be cased to a specific type * that is event-specific (but many events will just use NULL since they do not * have additional information to pass to the callback). * * If the callback is NULL and there was a previous subscription, the module * will be unsubscribed. If there was a previous subscription and the callback * is not null, the old callback will be replaced with the new one. * * The callback must be of this type: * * int (*ValkeyModuleEventCallback)(ValkeyModuleCtx *ctx, * ValkeyModuleEvent eid, * uint64_t subevent, * void *data); * * The 'ctx' is a normal module context that the callback can use in * order to call other modules APIs. The 'eid' is the event itself, this * is only useful in the case the module subscribed to multiple events: using * the 'id' field of this structure it is possible to check if the event * is one of the events we registered with this callback. The 'subevent' field * depends on the event that fired. * * Finally the 'data' pointer may be populated, only for certain events, with * more relevant data. * * Here is a list of events you can use as 'eid' and related sub events: * * * ValkeyModuleEvent_ReplicationRoleChanged: * * This event is called when the instance switches from primary * to replica or the other way around, however the event is * also called when the replica remains a replica but starts to * replicate with a different primary. * * The following sub events are available: * * * `VALKEYMODULE_SUBEVENT_REPLROLECHANGED_NOW_PRIMARY` * * `VALKEYMODULE_SUBEVENT_REPLROLECHANGED_NOW_REPLICA` * * The 'data' field can be casted by the callback to a * `ValkeyModuleReplicationInfo` structure with the following fields: * * int primary; // true if primary, false if replica * char *primary_host; // primary instance hostname for NOW_REPLICA * int primary_port; // primary instance port for NOW_REPLICA * char *replid1; // Main replication ID * char *replid2; // Secondary replication ID * uint64_t repl1_offset; // Main replication offset * uint64_t repl2_offset; // Offset of replid2 validity * * * ValkeyModuleEvent_Persistence * * This event is called when RDB saving or AOF rewriting starts * and ends. The following sub events are available: * * * `VALKEYMODULE_SUBEVENT_PERSISTENCE_RDB_START` * * `VALKEYMODULE_SUBEVENT_PERSISTENCE_AOF_START` * * `VALKEYMODULE_SUBEVENT_PERSISTENCE_SYNC_RDB_START` * * `VALKEYMODULE_SUBEVENT_PERSISTENCE_SYNC_AOF_START` * * `VALKEYMODULE_SUBEVENT_PERSISTENCE_ENDED` * * `VALKEYMODULE_SUBEVENT_PERSISTENCE_FAILED` * * The above events are triggered not just when the user calls the * relevant commands like BGSAVE, but also when a saving operation * or AOF rewriting occurs because of internal server triggers. * The SYNC_RDB_START sub events are happening in the foreground due to * SAVE command, FLUSHALL, or server shutdown, and the other RDB and * AOF sub events are executed in a background fork child, so any * action the module takes can only affect the generated AOF or RDB, * but will not be reflected in the parent process and affect connected * clients and commands. Also note that the AOF_START sub event may end * up saving RDB content in case of an AOF with rdb-preamble. * * * ValkeyModuleEvent_FlushDB * * The FLUSHALL, FLUSHDB or an internal flush (for instance * because of replication, after the replica synchronization) * happened. The following sub events are available: * * * `VALKEYMODULE_SUBEVENT_FLUSHDB_START` * * `VALKEYMODULE_SUBEVENT_FLUSHDB_END` * * The data pointer can be casted to a ValkeyModuleFlushInfo * structure with the following fields: * * int32_t async; // True if the flush is done in a thread. * // See for instance FLUSHALL ASYNC. * // In this case the END callback is invoked * // immediately after the database is put * // in the free list of the thread. * int32_t dbnum; // Flushed database number, -1 for all the DBs * // in the case of the FLUSHALL operation. * * The start event is called *before* the operation is initiated, thus * allowing the callback to call DBSIZE or other operation on the * yet-to-free keyspace. * * * ValkeyModuleEvent_Loading * * Called on loading operations: at startup when the server is * started, but also after a first synchronization when the * replica is loading the RDB file from the primary. * The following sub events are available: * * * `VALKEYMODULE_SUBEVENT_LOADING_RDB_START` * * `VALKEYMODULE_SUBEVENT_LOADING_AOF_START` * * `VALKEYMODULE_SUBEVENT_LOADING_REPL_START` * * `VALKEYMODULE_SUBEVENT_LOADING_ENDED` * * `VALKEYMODULE_SUBEVENT_LOADING_FAILED` * * Note that AOF loading may start with an RDB data in case of * rdb-preamble, in which case you'll only receive an AOF_START event. * * * ValkeyModuleEvent_ClientChange * * Called when a client connects or disconnects. * The data pointer can be casted to a ValkeyModuleClientInfo * structure, documented in ValkeyModule_GetClientInfoById(). * The following sub events are available: * * * `VALKEYMODULE_SUBEVENT_CLIENT_CHANGE_CONNECTED` * * `VALKEYMODULE_SUBEVENT_CLIENT_CHANGE_DISCONNECTED` * * * ValkeyModuleEvent_Shutdown * * The server is shutting down. No subevents are available. * * * ValkeyModuleEvent_ReplicaChange * * This event is called when the instance (that can be both a * primary or a replica) get a new online replica, or lose a * replica since it gets disconnected. * The following sub events are available: * * * `VALKEYMODULE_SUBEVENT_REPLICA_CHANGE_ONLINE` * * `VALKEYMODULE_SUBEVENT_REPLICA_CHANGE_OFFLINE` * * No additional information is available so far: future versions * of the server will have an API in order to enumerate the replicas * connected and their state. * * * ValkeyModuleEvent_CronLoop * * This event is called every time the server calls the serverCron() * function in order to do certain bookkeeping. Modules that are * required to do operations from time to time may use this callback. * Normally the server calls this function 10 times per second, but * this changes depending on the "hz" configuration. * No sub events are available. * * The data pointer can be casted to a ValkeyModuleCronLoop * structure with the following fields: * * int32_t hz; // Approximate number of events per second. * * * ValkeyModuleEvent_PrimaryLinkChange * * This is called for replicas in order to notify when the * replication link becomes functional (up) with our primary, * or when it goes down. Note that the link is not considered * up when we just connected to the primary, but only if the * replication is happening correctly. * The following sub events are available: * * * `VALKEYMODULE_SUBEVENT_PRIMARY_LINK_UP` * * `VALKEYMODULE_SUBEVENT_PRIMARY_LINK_DOWN` * * * ValkeyModuleEvent_ModuleChange * * This event is called when a new module is loaded or one is unloaded. * The following sub events are available: * * * `VALKEYMODULE_SUBEVENT_MODULE_LOADED` * * `VALKEYMODULE_SUBEVENT_MODULE_UNLOADED` * * The data pointer can be casted to a ValkeyModuleModuleChange * structure with the following fields: * * const char* module_name; // Name of module loaded or unloaded. * int32_t module_version; // Module version. * * * ValkeyModuleEvent_LoadingProgress * * This event is called repeatedly called while an RDB or AOF file * is being loaded. * The following sub events are available: * * * `VALKEYMODULE_SUBEVENT_LOADING_PROGRESS_RDB` * * `VALKEYMODULE_SUBEVENT_LOADING_PROGRESS_AOF` * * The data pointer can be casted to a ValkeyModuleLoadingProgress * structure with the following fields: * * int32_t hz; // Approximate number of events per second. * int32_t progress; // Approximate progress between 0 and 1024, * // or -1 if unknown. * * * ValkeyModuleEvent_SwapDB * * This event is called when a SWAPDB command has been successfully * Executed. * For this event call currently there is no subevents available. * * The data pointer can be casted to a ValkeyModuleSwapDbInfo * structure with the following fields: * * int32_t dbnum_first; // Swap Db first dbnum * int32_t dbnum_second; // Swap Db second dbnum * * * ValkeyModuleEvent_ReplBackup * * WARNING: Replication Backup events are deprecated since Redis OSS 7.0 and are never fired. * See ValkeyModuleEvent_ReplAsyncLoad for understanding how Async Replication Loading events * are now triggered when repl-diskless-load is set to swapdb. * * Called when repl-diskless-load config is set to swapdb, * And the server needs to backup the current database for the * possibility to be restored later. A module with global data and * maybe with aux_load and aux_save callbacks may need to use this * notification to backup / restore / discard its globals. * The following sub events are available: * * * `VALKEYMODULE_SUBEVENT_REPL_BACKUP_CREATE` * * `VALKEYMODULE_SUBEVENT_REPL_BACKUP_RESTORE` * * `VALKEYMODULE_SUBEVENT_REPL_BACKUP_DISCARD` * * * ValkeyModuleEvent_ReplAsyncLoad * * Called when repl-diskless-load config is set to swapdb and a replication with a primary of same * data set history (matching replication ID) occurs. * In which case the server serves current data set while loading new database in memory from socket. * Modules must have declared they support this mechanism in order to activate it, through * VALKEYMODULE_OPTIONS_HANDLE_REPL_ASYNC_LOAD flag. * The following sub events are available: * * * `VALKEYMODULE_SUBEVENT_REPL_ASYNC_LOAD_STARTED` * * `VALKEYMODULE_SUBEVENT_REPL_ASYNC_LOAD_ABORTED` * * `VALKEYMODULE_SUBEVENT_REPL_ASYNC_LOAD_COMPLETED` * * * ValkeyModuleEvent_ForkChild * * Called when a fork child (AOFRW, RDBSAVE, module fork...) is born/dies * The following sub events are available: * * * `VALKEYMODULE_SUBEVENT_FORK_CHILD_BORN` * * `VALKEYMODULE_SUBEVENT_FORK_CHILD_DIED` * * * ValkeyModuleEvent_EventLoop * * Called on each event loop iteration, once just before the event loop goes * to sleep or just after it wakes up. * The following sub events are available: * * * `VALKEYMODULE_SUBEVENT_EVENTLOOP_BEFORE_SLEEP` * * `VALKEYMODULE_SUBEVENT_EVENTLOOP_AFTER_SLEEP` * * * ValkeyModule_Event_Config * * Called when a configuration event happens * The following sub events are available: * * * `VALKEYMODULE_SUBEVENT_CONFIG_CHANGE` * * The data pointer can be casted to a ValkeyModuleConfigChange * structure with the following fields: * * const char **config_names; // An array of C string pointers containing the * // name of each modified configuration item * uint32_t num_changes; // The number of elements in the config_names array * * * ValkeyModule_Event_Key * * Called when a key is removed from the keyspace. We can't modify any key in * the event. * The following sub events are available: * * * `VALKEYMODULE_SUBEVENT_KEY_DELETED` * * `VALKEYMODULE_SUBEVENT_KEY_EXPIRED` * * `VALKEYMODULE_SUBEVENT_KEY_EVICTED` * * `VALKEYMODULE_SUBEVENT_KEY_OVERWRITTEN` * * The data pointer can be casted to a ValkeyModuleKeyInfo * structure with the following fields: * * ValkeyModuleKey *key; // Key name * * The function returns VALKEYMODULE_OK if the module was successfully subscribed * for the specified event. If the API is called from a wrong context or unsupported event * is given then VALKEYMODULE_ERR is returned. */ int VM_SubscribeToServerEvent(ValkeyModuleCtx *ctx, ValkeyModuleEvent event, ValkeyModuleEventCallback callback) { ValkeyModuleEventListener *el; /* Protect in case of calls from contexts without a module reference. */ if (ctx->module == NULL) return VALKEYMODULE_ERR; if (event.id >= _VALKEYMODULE_EVENT_NEXT) return VALKEYMODULE_ERR; if (event.dataver > moduleEventVersions[event.id]) return VALKEYMODULE_ERR; /* Module compiled with a newer valkeymodule.h than we support */ /* Search an event matching this module and event ID. */ listIter li; listNode *ln; listRewind(ValkeyModule_EventListeners, &li); while ((ln = listNext(&li))) { el = ln->value; if (el->module == ctx->module && el->event.id == event.id) break; /* Matching event found. */ } /* Modify or remove the event listener if we already had one. */ if (ln) { if (callback == NULL) { listDelNode(ValkeyModule_EventListeners, ln); zfree(el); } else { el->callback = callback; /* Update the callback with the new one. */ } return VALKEYMODULE_OK; } /* No event found, we need to add a new one. */ el = zmalloc(sizeof(*el)); el->module = ctx->module; el->event = event; el->callback = callback; listAddNodeTail(ValkeyModule_EventListeners, el); return VALKEYMODULE_OK; } /** * For a given server event and subevent, return zero if the * subevent is not supported and non-zero otherwise. */ int VM_IsSubEventSupported(ValkeyModuleEvent event, int64_t subevent) { switch (event.id) { case VALKEYMODULE_EVENT_REPLICATION_ROLE_CHANGED: return subevent < _VALKEYMODULE_EVENT_REPLROLECHANGED_NEXT; case VALKEYMODULE_EVENT_PERSISTENCE: return subevent < _VALKEYMODULE_SUBEVENT_PERSISTENCE_NEXT; case VALKEYMODULE_EVENT_FLUSHDB: return subevent < _VALKEYMODULE_SUBEVENT_FLUSHDB_NEXT; case VALKEYMODULE_EVENT_LOADING: return subevent < _VALKEYMODULE_SUBEVENT_LOADING_NEXT; case VALKEYMODULE_EVENT_CLIENT_CHANGE: return subevent < _VALKEYMODULE_SUBEVENT_CLIENT_CHANGE_NEXT; case VALKEYMODULE_EVENT_SHUTDOWN: return subevent < _VALKEYMODULE_SUBEVENT_SHUTDOWN_NEXT; case VALKEYMODULE_EVENT_REPLICA_CHANGE: return subevent < _VALKEYMODULE_EVENT_REPLROLECHANGED_NEXT; case VALKEYMODULE_EVENT_PRIMARY_LINK_CHANGE: return subevent < _VALKEYMODULE_SUBEVENT_PRIMARY_NEXT; case VALKEYMODULE_EVENT_CRON_LOOP: return subevent < _VALKEYMODULE_SUBEVENT_CRON_LOOP_NEXT; case VALKEYMODULE_EVENT_MODULE_CHANGE: return subevent < _VALKEYMODULE_SUBEVENT_MODULE_NEXT; case VALKEYMODULE_EVENT_LOADING_PROGRESS: return subevent < _VALKEYMODULE_SUBEVENT_LOADING_PROGRESS_NEXT; case VALKEYMODULE_EVENT_SWAPDB: return subevent < _VALKEYMODULE_SUBEVENT_SWAPDB_NEXT; case VALKEYMODULE_EVENT_REPL_ASYNC_LOAD: return subevent < _VALKEYMODULE_SUBEVENT_REPL_ASYNC_LOAD_NEXT; case VALKEYMODULE_EVENT_FORK_CHILD: return subevent < _VALKEYMODULE_SUBEVENT_FORK_CHILD_NEXT; case VALKEYMODULE_EVENT_EVENTLOOP: return subevent < _VALKEYMODULE_SUBEVENT_EVENTLOOP_NEXT; case VALKEYMODULE_EVENT_CONFIG: return subevent < _VALKEYMODULE_SUBEVENT_CONFIG_NEXT; case VALKEYMODULE_EVENT_KEY: return subevent < _VALKEYMODULE_SUBEVENT_KEY_NEXT; default: break; } return 0; } typedef struct KeyInfo { int32_t dbnum; ValkeyModuleString *key; robj *value; int mode; } KeyInfo; /* This is called by the server internals every time we want to fire an * event that can be intercepted by some module. The pointer 'data' is useful * in order to populate the event-specific structure when needed, in order * to return the structure with more information to the callback. * * 'eid' and 'subid' are just the main event ID and the sub event associated * with the event, depending on what exactly happened. */ void moduleFireServerEvent(uint64_t eid, int subid, void *data) { /* Fast path to return ASAP if there is nothing to do, avoiding to * setup the iterator and so forth: we want this call to be extremely * cheap if there are no registered modules. */ if (listLength(ValkeyModule_EventListeners) == 0) return; listIter li; listNode *ln; listRewind(ValkeyModule_EventListeners, &li); while ((ln = listNext(&li))) { ValkeyModuleEventListener *el = ln->value; if (el->event.id == eid) { ValkeyModuleCtx ctx; if (eid == VALKEYMODULE_EVENT_CLIENT_CHANGE) { /* In the case of client changes, we're pushing the real client * so the event handler can mutate it if needed. For example, * to change its authentication state in a way that does not * depend on specific commands executed later. */ moduleCreateContext(&ctx, el->module, VALKEYMODULE_CTX_NONE); ctx.client = (client *)data; } else { moduleCreateContext(&ctx, el->module, VALKEYMODULE_CTX_TEMP_CLIENT); } void *moduledata = NULL; ValkeyModuleClientInfoV1 civ1; ValkeyModuleReplicationInfoV1 riv1; ValkeyModuleModuleChangeV1 mcv1; ValkeyModuleKey key; ValkeyModuleKeyInfoV1 ki = {VALKEYMODULE_KEYINFO_VERSION, &key}; /* Event specific context and data pointer setup. */ if (eid == VALKEYMODULE_EVENT_CLIENT_CHANGE) { serverAssert(modulePopulateClientInfoStructure(&civ1, data, el->event.dataver) == VALKEYMODULE_OK); moduledata = &civ1; } else if (eid == VALKEYMODULE_EVENT_REPLICATION_ROLE_CHANGED) { serverAssert(modulePopulateReplicationInfoStructure(&riv1, el->event.dataver) == VALKEYMODULE_OK); moduledata = &riv1; } else if (eid == VALKEYMODULE_EVENT_FLUSHDB) { moduledata = data; ValkeyModuleFlushInfoV1 *fi = data; if (fi->dbnum != -1) selectDb(ctx.client, fi->dbnum); } else if (eid == VALKEYMODULE_EVENT_MODULE_CHANGE) { ValkeyModule *m = data; if (m == el->module) { moduleFreeContext(&ctx); continue; } mcv1.version = VALKEYMODULE_MODULE_CHANGE_VERSION; mcv1.module_name = m->name; mcv1.module_version = m->ver; moduledata = &mcv1; } else if (eid == VALKEYMODULE_EVENT_LOADING_PROGRESS) { moduledata = data; } else if (eid == VALKEYMODULE_EVENT_CRON_LOOP) { moduledata = data; } else if (eid == VALKEYMODULE_EVENT_SWAPDB) { moduledata = data; } else if (eid == VALKEYMODULE_EVENT_CONFIG) { moduledata = data; } else if (eid == VALKEYMODULE_EVENT_KEY) { KeyInfo *info = data; selectDb(ctx.client, info->dbnum); moduleInitKey(&key, &ctx, info->key, info->value, info->mode); moduledata = &ki; } el->module->in_hook++; el->callback(&ctx, el->event, subid, moduledata); el->module->in_hook--; if (eid == VALKEYMODULE_EVENT_KEY) { moduleCloseKey(&key); } moduleFreeContext(&ctx); } } } /* Remove all the listeners for this module: this is used before unloading * a module. */ void moduleUnsubscribeAllServerEvents(ValkeyModule *module) { ValkeyModuleEventListener *el; listIter li; listNode *ln; listRewind(ValkeyModule_EventListeners, &li); while ((ln = listNext(&li))) { el = ln->value; if (el->module == module) { listDelNode(ValkeyModule_EventListeners, ln); zfree(el); } } } void processModuleLoadingProgressEvent(int is_aof) { long long now = server.ustime; static long long next_event = 0; if (now >= next_event) { /* Fire the loading progress modules end event. */ int progress = -1; if (server.loading_total_bytes) progress = (server.loading_loaded_bytes << 10) / server.loading_total_bytes; ValkeyModuleLoadingProgressV1 fi = {VALKEYMODULE_LOADING_PROGRESS_VERSION, server.hz, progress}; moduleFireServerEvent( VALKEYMODULE_EVENT_LOADING_PROGRESS, is_aof ? VALKEYMODULE_SUBEVENT_LOADING_PROGRESS_AOF : VALKEYMODULE_SUBEVENT_LOADING_PROGRESS_RDB, &fi); /* decide when the next event should fire. */ next_event = now + 1000000 / server.hz; } } /* When a key is deleted (in dbAsyncDelete/dbSyncDelete/setKey), it * will be called to tell the module which key is about to be released. */ void moduleNotifyKeyUnlink(robj *key, robj *val, int dbid, int flags) { server.lazy_expire_disabled++; int subevent = VALKEYMODULE_SUBEVENT_KEY_DELETED; if (flags & DB_FLAG_KEY_EXPIRED) { subevent = VALKEYMODULE_SUBEVENT_KEY_EXPIRED; } else if (flags & DB_FLAG_KEY_EVICTED) { subevent = VALKEYMODULE_SUBEVENT_KEY_EVICTED; } else if (flags & DB_FLAG_KEY_OVERWRITE) { subevent = VALKEYMODULE_SUBEVENT_KEY_OVERWRITTEN; } KeyInfo info = {dbid, key, val, VALKEYMODULE_READ}; moduleFireServerEvent(VALKEYMODULE_EVENT_KEY, subevent, &info); if (val->type == OBJ_MODULE) { moduleValue *mv = val->ptr; moduleType *mt = mv->type; /* We prefer to use the enhanced version. */ if (mt->unlink2 != NULL) { ValkeyModuleKeyOptCtx ctx = {key, NULL, dbid, -1}; mt->unlink2(&ctx, mv->value); } else if (mt->unlink != NULL) { mt->unlink(key, mv->value); } } server.lazy_expire_disabled--; } /* Return the free_effort of the module, it will automatically choose to call * `free_effort` or `free_effort2`, and the default return value is 1. * value of 0 means very high effort (always asynchronous freeing). */ size_t moduleGetFreeEffort(robj *key, robj *val, int dbid) { moduleValue *mv = val->ptr; moduleType *mt = mv->type; size_t effort = 1; /* We prefer to use the enhanced version. */ if (mt->free_effort2 != NULL) { ValkeyModuleKeyOptCtx ctx = {key, NULL, dbid, -1}; effort = mt->free_effort2(&ctx, mv->value); } else if (mt->free_effort != NULL) { effort = mt->free_effort(key, mv->value); } return effort; } /* Return the memory usage of the module, it will automatically choose to call * `mem_usage` or `mem_usage2`, and the default return value is 0. */ size_t moduleGetMemUsage(robj *key, robj *val, size_t sample_size, int dbid) { moduleValue *mv = val->ptr; moduleType *mt = mv->type; size_t size = 0; /* We prefer to use the enhanced version. */ if (mt->mem_usage2 != NULL) { ValkeyModuleKeyOptCtx ctx = {key, NULL, dbid, -1}; size = mt->mem_usage2(&ctx, mv->value, sample_size); } else if (mt->mem_usage != NULL) { size = mt->mem_usage(mv->value); } return size; } /* -------------------------------------------------------------------------- * Modules API internals * -------------------------------------------------------------------------- */ /* server.moduleapi dictionary type. Only uses plain C strings since * this gets queries from modules. */ uint64_t dictCStringKeyHash(const void *key) { return dictGenHashFunction((unsigned char *)key, strlen((char *)key)); } int dictCStringKeyCompare(const void *key1, const void *key2) { return strcmp(key1, key2) == 0; } dictType moduleAPIDictType = { dictCStringKeyHash, /* hash function */ NULL, /* key dup */ dictCStringKeyCompare, /* key compare */ NULL, /* key destructor */ NULL, /* val destructor */ NULL /* allow to expand */ }; int moduleRegisterApi(const char *funcname, void *funcptr) { return dictAdd(server.moduleapi, (char *)funcname, funcptr); } /* Register Module APIs under both RedisModule_ and ValkeyModule_ namespaces * so that legacy Redis module binaries can continue to function */ #define REGISTER_API(name) \ moduleRegisterApi("ValkeyModule_" #name, (void *)(unsigned long)VM_##name); \ moduleRegisterApi("RedisModule_" #name, (void *)(unsigned long)VM_##name); /* Global initialization at server startup. */ void moduleRegisterCoreAPI(void); /* Currently, this function is just a placeholder for the module system * initialization steps that need to be run after server initialization. * A previous issue, selectDb() in createClient() requires that server.db has * been initialized, see #7323. */ void moduleInitModulesSystemLast(void) { } dictType sdsKeyValueHashDictType = { dictSdsCaseHash, /* hash function */ NULL, /* key dup */ dictSdsKeyCaseCompare, /* key compare */ dictSdsDestructor, /* key destructor */ dictSdsDestructor, /* val destructor */ NULL /* allow to expand */ }; void moduleInitModulesSystem(void) { moduleUnblockedClients = listCreate(); server.loadmodule_queue = listCreate(); server.module_configs_queue = dictCreate(&sdsKeyValueHashDictType); server.module_gil_acquiring = 0; modules = dictCreate(&modulesDictType); moduleAuthCallbacks = listCreate(); /* Set up the keyspace notification subscriber list and static client */ moduleKeyspaceSubscribers = listCreate(); modulePostExecUnitJobs = listCreate(); /* Set up filter list */ moduleCommandFilters = listCreate(); moduleRegisterCoreAPI(); /* Create a pipe for module threads to be able to wake up the server main thread. * Make the pipe non blocking. This is just a best effort aware mechanism * and we do not want to block not in the read nor in the write half. * Enable close-on-exec flag on pipes in case of the fork-exec system calls in * sentinels or servers. */ if (anetPipe(server.module_pipe, O_CLOEXEC | O_NONBLOCK, O_CLOEXEC | O_NONBLOCK) == -1) { serverLog(LL_WARNING, "Can't create the pipe for module threads: %s", strerror(errno)); exit(1); } /* Create the timers radix tree. */ Timers = raxNew(); /* Setup the event listeners data structures. */ ValkeyModule_EventListeners = listCreate(); /* Making sure moduleEventVersions is synced with the number of events. */ serverAssert(sizeof(moduleEventVersions) / sizeof(moduleEventVersions[0]) == _VALKEYMODULE_EVENT_NEXT); /* Our thread-safe contexts GIL must start with already locked: * it is just unlocked when it's safe. */ pthread_mutex_lock(&moduleGIL); } void modulesCron(void) { /* Check number of temporary clients in the pool and free the unused ones * since the last cron. moduleTempClientMinCount tracks minimum count of * clients in the pool since the last cron. This is the number of clients * that we didn't use for the last cron period. */ /* Limit the max client count to be freed at once to avoid latency spikes.*/ int iteration = 50; /* We are freeing clients if we have more than 8 unused clients. Keeping * small amount of clients to avoid client allocation costs if temporary * clients are required after some idle period. */ const unsigned int min_client = 8; while (iteration > 0 && moduleTempClientCount > 0 && moduleTempClientMinCount > min_client) { client *c = moduleTempClients[--moduleTempClientCount]; freeClient(c); iteration--; moduleTempClientMinCount--; } moduleTempClientMinCount = moduleTempClientCount; /* Shrink moduleTempClients array itself if it is wasting some space */ if (moduleTempClientCap > 32 && moduleTempClientCap > moduleTempClientCount * 4) { moduleTempClientCap /= 4; moduleTempClients = zrealloc(moduleTempClients, sizeof(client *) * moduleTempClientCap); } } void moduleLoadQueueEntryFree(struct moduleLoadQueueEntry *loadmod) { if (!loadmod) return; sdsfree(loadmod->path); for (int i = 0; i < loadmod->argc; i++) { decrRefCount(loadmod->argv[i]); } zfree(loadmod->argv); zfree(loadmod); } /* Remove Module Configs from standardConfig array in config.c */ void moduleRemoveConfigs(ValkeyModule *module) { listIter li; listNode *ln; listRewind(module->module_configs, &li); while ((ln = listNext(&li))) { ModuleConfig *config = listNodeValue(ln); sds module_name = sdsnew(module->name); sds full_name = sdscat(sdscat(module_name, "."), config->name); /* ModuleName.ModuleConfig */ removeConfig(full_name); sdsfree(full_name); } } /* Remove ACL categories added by the module when it fails to load. */ void moduleRemoveCateogires(ValkeyModule *module) { if (module->num_acl_categories_added) { ACLCleanupCategoriesOnFailure(module->num_acl_categories_added); } } /* Load all the modules in the server.loadmodule_queue list, which is * populated by `loadmodule` directives in the configuration file. * We can't load modules directly when processing the configuration file * because the server must be fully initialized before loading modules. * * The function aborts the server on errors, since to start with missing * modules is not considered sane: clients may rely on the existence of * given commands, loading AOF also may need some modules to exist, and * if this instance is a replica, it must understand commands from primary. */ void moduleLoadFromQueue(void) { listIter li; listNode *ln; listRewind(server.loadmodule_queue, &li); while ((ln = listNext(&li))) { struct moduleLoadQueueEntry *loadmod = ln->value; if (moduleLoad(loadmod->path, (void **)loadmod->argv, loadmod->argc, 0) == C_ERR) { serverLog(LL_WARNING, "Can't load module from %s: server aborting", loadmod->path); exit(1); } moduleLoadQueueEntryFree(loadmod); listDelNode(server.loadmodule_queue, ln); } if (dictSize(server.module_configs_queue)) { dictIterator *di = dictGetSafeIterator(server.module_configs_queue); dictEntry *de; while ((de = dictNext(di)) != NULL) { const char *moduleConfigName = dictGetKey(de); serverLog(LL_WARNING, "Unused Module Configuration: %s", moduleConfigName); } dictReleaseIterator(di); serverLog(LL_WARNING, "Module Configuration detected without loadmodule directive or no ApplyConfig call: aborting"); exit(1); } } void moduleFreeModuleStructure(struct ValkeyModule *module) { listRelease(module->types); listRelease(module->filters); listRelease(module->usedby); listRelease(module->using); listRelease(module->module_configs); sdsfree(module->name); moduleLoadQueueEntryFree(module->loadmod); zfree(module); } void moduleFreeArgs(struct serverCommandArg *args, int num_args) { for (int j = 0; j < num_args; j++) { zfree((char *)args[j].name); zfree((char *)args[j].token); zfree((char *)args[j].summary); zfree((char *)args[j].since); zfree((char *)args[j].deprecated_since); zfree((char *)args[j].display_text); if (args[j].subargs) { moduleFreeArgs(args[j].subargs, args[j].num_args); } } zfree(args); } /* Free the command registered with the specified module. * On success C_OK is returned, otherwise C_ERR is returned. * * Note that caller needs to handle the deletion of the command table dict, * and after that needs to free the command->fullname and the command itself. */ int moduleFreeCommand(struct ValkeyModule *module, struct serverCommand *cmd) { if (cmd->proc != ValkeyModuleCommandDispatcher) return C_ERR; ValkeyModuleCommand *cp = cmd->module_cmd; if (cp->module != module) return C_ERR; /* Free everything except cmd->fullname and cmd itself. */ for (int j = 0; j < cmd->key_specs_num; j++) { if (cmd->key_specs[j].notes) zfree((char *)cmd->key_specs[j].notes); if (cmd->key_specs[j].begin_search_type == KSPEC_BS_KEYWORD) zfree((char *)cmd->key_specs[j].bs.keyword.keyword); } zfree(cmd->key_specs); for (int j = 0; cmd->tips && cmd->tips[j]; j++) zfree((char *)cmd->tips[j]); zfree(cmd->tips); for (int j = 0; cmd->history && cmd->history[j].since; j++) { zfree((char *)cmd->history[j].since); zfree((char *)cmd->history[j].changes); } zfree(cmd->history); zfree((char *)cmd->summary); zfree((char *)cmd->since); zfree((char *)cmd->deprecated_since); zfree((char *)cmd->complexity); if (cmd->latency_histogram) { hdr_close(cmd->latency_histogram); cmd->latency_histogram = NULL; } moduleFreeArgs(cmd->args, cmd->num_args); zfree(cp); if (cmd->subcommands_ht) { hashtableIterator iter; void *next; hashtableInitIterator(&iter, cmd->subcommands_ht, HASHTABLE_ITER_SAFE); while (hashtableNext(&iter, &next)) { struct serverCommand *sub = next; if (moduleFreeCommand(module, sub) != C_OK) continue; serverAssert(hashtableDelete(cmd->subcommands_ht, sub->declared_name)); sdsfree((sds)sub->declared_name); sdsfree(sub->fullname); zfree(sub); } hashtableResetIterator(&iter); hashtableRelease(cmd->subcommands_ht); } return C_OK; } void moduleUnregisterCommands(struct ValkeyModule *module) { /* Drain IO queue before modifying commands dictionary to prevent concurrent access while modifying it. */ drainIOThreadsQueue(); /* Unregister all the commands registered by this module. */ hashtableIterator iter; void *next; hashtableInitIterator(&iter, server.commands, HASHTABLE_ITER_SAFE); while (hashtableNext(&iter, &next)) { struct serverCommand *cmd = next; if (moduleFreeCommand(module, cmd) != C_OK) continue; serverAssert(hashtableDelete(server.commands, cmd->fullname)); serverAssert(hashtableDelete(server.orig_commands, cmd->fullname)); sdsfree((sds)cmd->declared_name); sdsfree(cmd->fullname); zfree(cmd); } hashtableResetIterator(&iter); } /* We parse argv to add sds "NAME VALUE" pairs to the server.module_configs_queue list of configs. * We also increment the module_argv pointer to just after ARGS if there are args, otherwise * we set it to NULL */ int parseLoadexArguments(ValkeyModuleString ***module_argv, int *module_argc) { int args_specified = 0; ValkeyModuleString **argv = *module_argv; int argc = *module_argc; for (int i = 0; i < argc; i++) { char *arg_val = argv[i]->ptr; if (!strcasecmp(arg_val, "CONFIG")) { if (i + 2 >= argc) { serverLog(LL_NOTICE, "CONFIG specified without name value pair"); return VALKEYMODULE_ERR; } sds name = sdsdup(argv[i + 1]->ptr); sds value = sdsdup(argv[i + 2]->ptr); if (!dictReplace(server.module_configs_queue, name, value)) sdsfree(name); i += 2; } else if (!strcasecmp(arg_val, "ARGS")) { args_specified = 1; i++; if (i >= argc) { *module_argv = NULL; *module_argc = 0; } else { *module_argv = argv + i; *module_argc = argc - i; } break; } else { serverLog(LL_NOTICE, "Syntax Error from arguments to loadex around %s.", arg_val); return VALKEYMODULE_ERR; } } if (!args_specified) { *module_argv = NULL; *module_argc = 0; } return VALKEYMODULE_OK; } /* Unregister module-related things, called when moduleLoad fails or moduleUnload. */ void moduleUnregisterCleanup(ValkeyModule *module) { moduleFreeAuthenticatedClients(module); moduleUnregisterCommands(module); moduleUnsubscribeNotifications(module); moduleUnregisterSharedAPI(module); moduleUnregisterUsedAPI(module); moduleUnregisterFilters(module); moduleUnsubscribeAllServerEvents(module); moduleRemoveConfigs(module); moduleUnregisterAuthCBs(module); } /* Load a module and initialize it. On success C_OK is returned, otherwise * C_ERR is returned. */ int moduleLoad(const char *path, void **module_argv, int module_argc, int is_loadex) { int (*onload)(void *, void **, int); void *handle; struct stat st; if (stat(path, &st) == 0) { /* This check is best effort */ if (!(st.st_mode & (S_IXUSR | S_IXGRP | S_IXOTH))) { serverLog(LL_WARNING, "Module %s failed to load: It does not have execute permissions.", path); return C_ERR; } } handle = dlopen(path, RTLD_NOW | RTLD_LOCAL); if (handle == NULL) { serverLog(LL_WARNING, "Module %s failed to load: %s", path, dlerror()); return C_ERR; } const char *onLoadNames[] = {"ValkeyModule_OnLoad", "RedisModule_OnLoad"}; for (size_t i = 0; i < sizeof(onLoadNames) / sizeof(onLoadNames[0]); i++) { onload = (int (*)(void *, void **, int))(unsigned long)dlsym(handle, onLoadNames[i]); if (onload != NULL) { if (i != 0) { serverLog(LL_NOTICE, "Legacy Redis Module %s found", path); } break; } } if (onload == NULL) { dlclose(handle); serverLog(LL_WARNING, "Module %s does not export ValkeyModule_OnLoad() or RedisModule_OnLoad() " "symbol. Module not loaded.", path); return C_ERR; } ValkeyModuleCtx ctx; moduleCreateContext(&ctx, NULL, VALKEYMODULE_CTX_TEMP_CLIENT); /* We pass NULL since we don't have a module yet. */ if (onload((void *)&ctx, module_argv, module_argc) == VALKEYMODULE_ERR) { if (ctx.module) { serverLog(LL_WARNING, "Module %s initialization failed. Module not loaded.", path); moduleUnregisterCleanup(ctx.module); moduleRemoveCateogires(ctx.module); moduleFreeModuleStructure(ctx.module); } else { /* If there is no ctx.module, this means that our ValkeyModule_Init call failed, * and currently init will only fail on busy name. */ serverLog(LL_WARNING, "Module %s initialization failed. Module name is busy.", path); } moduleFreeContext(&ctx); dlclose(handle); return C_ERR; } /* Module loaded! Register it. */ dictAdd(modules, ctx.module->name, ctx.module); ctx.module->blocked_clients = 0; ctx.module->handle = handle; ctx.module->loadmod = zmalloc(sizeof(struct moduleLoadQueueEntry)); ctx.module->loadmod->path = sdsnew(path); ctx.module->loadmod->argv = module_argc ? zmalloc(sizeof(robj *) * module_argc) : NULL; ctx.module->loadmod->argc = module_argc; for (int i = 0; i < module_argc; i++) { ctx.module->loadmod->argv[i] = module_argv[i]; incrRefCount(ctx.module->loadmod->argv[i]); } /* If module commands have ACL categories, recompute command bits * for all existing users once the modules has been registered. */ if (ctx.module->num_commands_with_acl_categories) { ACLRecomputeCommandBitsFromCommandRulesAllUsers(); } serverLog(LL_NOTICE, "Module '%s' loaded from %s", ctx.module->name, path); ctx.module->onload = 0; int post_load_err = 0; if (listLength(ctx.module->module_configs) && !ctx.module->configs_initialized) { serverLogRaw(LL_WARNING, "Module Configurations were not set, likely a missing LoadConfigs call. Unloading the module."); post_load_err = 1; } if (is_loadex && dictSize(server.module_configs_queue)) { serverLogRaw(LL_WARNING, "Loadex configurations were not applied, likely due to invalid arguments. Unloading the module."); post_load_err = 1; } if (post_load_err) { moduleUnload(ctx.module->name, NULL); moduleFreeContext(&ctx); return C_ERR; } /* Fire the loaded modules event. */ moduleFireServerEvent(VALKEYMODULE_EVENT_MODULE_CHANGE, VALKEYMODULE_SUBEVENT_MODULE_LOADED, ctx.module); moduleFreeContext(&ctx); return C_OK; } /* Unload the module registered with the specified name. On success * C_OK is returned, otherwise C_ERR is returned and errmsg is set * with an appropriate message. */ int moduleUnload(sds name, const char **errmsg) { struct ValkeyModule *module = dictFetchValue(modules, name); if (module == NULL) { *errmsg = "no such module with that name"; return C_ERR; } else if (listLength(module->types)) { *errmsg = "the module exports one or more module-side data " "types, can't unload"; return C_ERR; } else if (listLength(module->usedby)) { *errmsg = "the module exports APIs used by other modules. " "Please unload them first and try again"; return C_ERR; } else if (module->blocked_clients) { *errmsg = "the module has blocked clients. " "Please wait for them to be unblocked and try again"; return C_ERR; } else if (moduleHoldsTimer(module)) { *errmsg = "the module holds timer that is not fired. " "Please stop the timer or wait until it fires."; return C_ERR; } /* Give module a chance to clean up. */ const char *onUnloadNames[] = {"ValkeyModule_OnUnload", "RedisModule_OnUnload"}; int (*onunload)(void *) = NULL; for (size_t i = 0; i < sizeof(onUnloadNames) / sizeof(onUnloadNames[0]); i++) { onunload = (int (*)(void *))(unsigned long)dlsym(module->handle, onUnloadNames[i]); if (onunload) { if (i != 0) { serverLog(LL_NOTICE, "Legacy Redis Module %s found", name); } break; } } if (onunload) { ValkeyModuleCtx ctx; moduleCreateContext(&ctx, module, VALKEYMODULE_CTX_TEMP_CLIENT); int unload_status = onunload((void *)&ctx); moduleFreeContext(&ctx); if (unload_status == VALKEYMODULE_ERR) { serverLog(LL_WARNING, "Module %s OnUnload failed. Unload canceled.", name); errno = ECANCELED; return C_ERR; } } moduleUnregisterCleanup(module); /* Unload the dynamic library. */ if (dlclose(module->handle) == -1) { char *error = dlerror(); if (error == NULL) error = "Unknown error"; serverLog(LL_WARNING, "Error when trying to close the %s module: %s", module->name, error); } /* Fire the unloaded modules event. */ moduleFireServerEvent(VALKEYMODULE_EVENT_MODULE_CHANGE, VALKEYMODULE_SUBEVENT_MODULE_UNLOADED, module); /* Remove from list of modules. */ serverLog(LL_NOTICE, "Module %s unloaded", module->name); dictDelete(modules, module->name); module->name = NULL; /* The name was already freed by dictDelete(). */ moduleFreeModuleStructure(module); /* Recompute command bits for all users once the modules has been completely unloaded. */ ACLRecomputeCommandBitsFromCommandRulesAllUsers(); return C_OK; } void modulePipeReadable(aeEventLoop *el, int fd, void *privdata, int mask) { UNUSED(el); UNUSED(fd); UNUSED(mask); UNUSED(privdata); char buf[128]; while (read(fd, buf, sizeof(buf)) == sizeof(buf)); /* Handle event loop events if pipe was written from event loop API */ eventLoopHandleOneShotEvents(); } /* Helper function for the MODULE and HELLO command: send the list of the * loaded modules to the client. */ void addReplyLoadedModules(client *c) { dictIterator *di = dictGetIterator(modules); dictEntry *de; addReplyArrayLen(c, dictSize(modules)); while ((de = dictNext(di)) != NULL) { sds name = dictGetKey(de); struct ValkeyModule *module = dictGetVal(de); sds path = module->loadmod->path; addReplyMapLen(c, 4); addReplyBulkCString(c, "name"); addReplyBulkCBuffer(c, name, sdslen(name)); addReplyBulkCString(c, "ver"); addReplyLongLong(c, module->ver); addReplyBulkCString(c, "path"); addReplyBulkCBuffer(c, path, sdslen(path)); addReplyBulkCString(c, "args"); addReplyArrayLen(c, module->loadmod->argc); for (int i = 0; i < module->loadmod->argc; i++) { addReplyBulk(c, module->loadmod->argv[i]); } } dictReleaseIterator(di); } /* Helper for genModulesInfoString(): given a list of modules, return * an SDS string in the form "[modulename|modulename2|...]" */ sds genModulesInfoStringRenderModulesList(list *l) { listIter li; listNode *ln; listRewind(l, &li); sds output = sdsnew("["); while ((ln = listNext(&li))) { ValkeyModule *module = ln->value; output = sdscat(output, module->name); if (ln != listLast(l)) output = sdscat(output, "|"); } output = sdscat(output, "]"); return output; } /* Helper for genModulesInfoString(): render module options as an SDS string. */ sds genModulesInfoStringRenderModuleOptions(struct ValkeyModule *module) { sds output = sdsnew("["); if (module->options & VALKEYMODULE_OPTIONS_HANDLE_IO_ERRORS) output = sdscat(output, "handle-io-errors|"); if (module->options & VALKEYMODULE_OPTIONS_HANDLE_REPL_ASYNC_LOAD) output = sdscat(output, "handle-repl-async-load|"); if (module->options & VALKEYMODULE_OPTION_NO_IMPLICIT_SIGNAL_MODIFIED) output = sdscat(output, "no-implicit-signal-modified|"); output = sdstrim(output, "|"); output = sdscat(output, "]"); return output; } /* Helper function for the INFO command: adds loaded modules as to info's * output. * * After the call, the passed sds info string is no longer valid and all the * * references must be substituted with the new pointer returned by the call. */ sds genModulesInfoString(sds info) { dictIterator *di = dictGetIterator(modules); dictEntry *de; while ((de = dictNext(di)) != NULL) { sds name = dictGetKey(de); struct ValkeyModule *module = dictGetVal(de); sds usedby = genModulesInfoStringRenderModulesList(module->usedby); sds using = genModulesInfoStringRenderModulesList(module->using); sds options = genModulesInfoStringRenderModuleOptions(module); info = sdscatfmt(info, "module:name=%S,ver=%i,api=%i,filters=%i," "usedby=%S,using=%S,options=%S\r\n", name, module->ver, module->apiver, (int)listLength(module->filters), usedby, using, options); sdsfree(usedby); sdsfree(using); sdsfree(options); } dictReleaseIterator(di); return info; } /* -------------------------------------------------------------------------- * Module Configurations API internals * -------------------------------------------------------------------------- */ /* Check if the configuration name is already registered */ int isModuleConfigNameRegistered(ValkeyModule *module, const char *name) { listNode *match = listSearchKey(module->module_configs, (void *)name); return match != NULL; } /* Assert that the flags passed into the VM_RegisterConfig Suite are valid */ int moduleVerifyConfigFlags(unsigned int flags, configType type) { if ((flags & ~(VALKEYMODULE_CONFIG_DEFAULT | VALKEYMODULE_CONFIG_IMMUTABLE | VALKEYMODULE_CONFIG_SENSITIVE | VALKEYMODULE_CONFIG_HIDDEN | VALKEYMODULE_CONFIG_PROTECTED | VALKEYMODULE_CONFIG_DENY_LOADING | VALKEYMODULE_CONFIG_BITFLAGS | VALKEYMODULE_CONFIG_MEMORY))) { serverLogRaw(LL_WARNING, "Invalid flag(s) for configuration"); return VALKEYMODULE_ERR; } if (type != NUMERIC_CONFIG && flags & VALKEYMODULE_CONFIG_MEMORY) { serverLogRaw(LL_WARNING, "Numeric flag provided for non-numeric configuration."); return VALKEYMODULE_ERR; } if (type != ENUM_CONFIG && flags & VALKEYMODULE_CONFIG_BITFLAGS) { serverLogRaw(LL_WARNING, "Enum flag provided for non-enum configuration."); return VALKEYMODULE_ERR; } return VALKEYMODULE_OK; } /* Verify a module resource or name has only alphanumeric characters, underscores * or dashes. */ int moduleVerifyResourceName(const char *name) { if (name[0] == '\0') { return VALKEYMODULE_ERR; } for (size_t i = 0; name[i] != '\0'; i++) { char curr_char = name[i]; if ((curr_char >= 'a' && curr_char <= 'z') || (curr_char >= 'A' && curr_char <= 'Z') || (curr_char >= '0' && curr_char <= '9') || (curr_char == '_') || (curr_char == '-')) { continue; } serverLog(LL_WARNING, "Invalid character %c in Module resource name %s.", curr_char, name); return VALKEYMODULE_ERR; } return VALKEYMODULE_OK; } /* This is a series of set functions for each type that act as dispatchers for * config.c to call module set callbacks. */ #define CONFIG_ERR_SIZE 256 static char configerr[CONFIG_ERR_SIZE]; static void propagateErrorString(ValkeyModuleString *err_in, const char **err) { if (err_in) { valkey_strlcpy(configerr, err_in->ptr, CONFIG_ERR_SIZE); decrRefCount(err_in); *err = configerr; } } int setModuleBoolConfig(ModuleConfig *config, int val, const char **err) { ValkeyModuleString *error = NULL; int return_code = config->set_fn.set_bool(config->name, val, config->privdata, &error); propagateErrorString(error, err); return return_code == VALKEYMODULE_OK ? 1 : 0; } int setModuleStringConfig(ModuleConfig *config, sds strval, const char **err) { ValkeyModuleString *error = NULL; ValkeyModuleString *new = createStringObject(strval, sdslen(strval)); int return_code = config->set_fn.set_string(config->name, new, config->privdata, &error); propagateErrorString(error, err); decrRefCount(new); return return_code == VALKEYMODULE_OK ? 1 : 0; } int setModuleEnumConfig(ModuleConfig *config, int val, const char **err) { ValkeyModuleString *error = NULL; int return_code = config->set_fn.set_enum(config->name, val, config->privdata, &error); propagateErrorString(error, err); return return_code == VALKEYMODULE_OK ? 1 : 0; } int setModuleNumericConfig(ModuleConfig *config, long long val, const char **err) { ValkeyModuleString *error = NULL; int return_code = config->set_fn.set_numeric(config->name, val, config->privdata, &error); propagateErrorString(error, err); return return_code == VALKEYMODULE_OK ? 1 : 0; } /* This is a series of get functions for each type that act as dispatchers for * config.c to call module set callbacks. */ int getModuleBoolConfig(ModuleConfig *module_config) { return module_config->get_fn.get_bool(module_config->name, module_config->privdata); } sds getModuleStringConfig(ModuleConfig *module_config) { ValkeyModuleString *val = module_config->get_fn.get_string(module_config->name, module_config->privdata); return val ? sdsdup(val->ptr) : NULL; } int getModuleEnumConfig(ModuleConfig *module_config) { return module_config->get_fn.get_enum(module_config->name, module_config->privdata); } long long getModuleNumericConfig(ModuleConfig *module_config) { return module_config->get_fn.get_numeric(module_config->name, module_config->privdata); } /* This function takes a module and a list of configs stored as sds NAME VALUE pairs. * It attempts to call set on each of these configs. */ int loadModuleConfigs(ValkeyModule *module) { listIter li; listNode *ln; const char *err = NULL; listRewind(module->module_configs, &li); while ((ln = listNext(&li))) { ModuleConfig *module_config = listNodeValue(ln); sds config_name = sdscatfmt(sdsempty(), "%s.%s", module->name, module_config->name); dictEntry *config_argument = dictFind(server.module_configs_queue, config_name); if (config_argument) { if (!performModuleConfigSetFromName(dictGetKey(config_argument), dictGetVal(config_argument), &err)) { serverLog(LL_WARNING, "Issue during loading of configuration %s : %s", (sds)dictGetKey(config_argument), err); sdsfree(config_name); dictEmpty(server.module_configs_queue, NULL); return VALKEYMODULE_ERR; } } else { if (!performModuleConfigSetDefaultFromName(config_name, &err)) { serverLog(LL_WARNING, "Issue attempting to set default value of configuration %s : %s", module_config->name, err); sdsfree(config_name); dictEmpty(server.module_configs_queue, NULL); return VALKEYMODULE_ERR; } } dictDelete(server.module_configs_queue, config_name); sdsfree(config_name); } module->configs_initialized = 1; return VALKEYMODULE_OK; } /* Add module_config to the list if the apply and privdata do not match one already in it. */ void addModuleConfigApply(list *module_configs, ModuleConfig *module_config) { if (!module_config->apply_fn) return; listIter li; listNode *ln; ModuleConfig *pending_apply; listRewind(module_configs, &li); while ((ln = listNext(&li))) { pending_apply = listNodeValue(ln); if (pending_apply->apply_fn == module_config->apply_fn && pending_apply->privdata == module_config->privdata) { return; } } listAddNodeTail(module_configs, module_config); } /* Call apply on all module configs specified in set, if an apply function was specified at registration time. */ int moduleConfigApplyConfig(list *module_configs, const char **err, const char **err_arg_name) { if (!listLength(module_configs)) return 1; listIter li; listNode *ln; ModuleConfig *module_config; ValkeyModuleString *error = NULL; ValkeyModuleCtx ctx; listRewind(module_configs, &li); while ((ln = listNext(&li))) { module_config = listNodeValue(ln); moduleCreateContext(&ctx, module_config->module, VALKEYMODULE_CTX_NONE); if (module_config->apply_fn(&ctx, module_config->privdata, &error)) { if (err_arg_name) *err_arg_name = module_config->name; propagateErrorString(error, err); moduleFreeContext(&ctx); return 0; } moduleFreeContext(&ctx); } return 1; } /* -------------------------------------------------------------------------- * ## Module Configurations API * -------------------------------------------------------------------------- */ /* Create a module config object. */ ModuleConfig * createModuleConfig(const char *name, ValkeyModuleConfigApplyFunc apply_fn, void *privdata, ValkeyModule *module) { ModuleConfig *new_config = zmalloc(sizeof(ModuleConfig)); new_config->name = sdsnew(name); new_config->apply_fn = apply_fn; new_config->privdata = privdata; new_config->module = module; return new_config; } int moduleConfigValidityCheck(ValkeyModule *module, const char *name, unsigned int flags, configType type) { if (!module->onload) { errno = EBUSY; return VALKEYMODULE_ERR; } if (moduleVerifyConfigFlags(flags, type) || moduleVerifyResourceName(name)) { errno = EINVAL; return VALKEYMODULE_ERR; } if (isModuleConfigNameRegistered(module, name)) { serverLog(LL_WARNING, "Configuration by the name: %s already registered", name); errno = EALREADY; return VALKEYMODULE_ERR; } return VALKEYMODULE_OK; } unsigned int maskModuleConfigFlags(unsigned int flags) { unsigned int new_flags = 0; if (flags & VALKEYMODULE_CONFIG_DEFAULT) new_flags |= MODIFIABLE_CONFIG; if (flags & VALKEYMODULE_CONFIG_IMMUTABLE) new_flags |= IMMUTABLE_CONFIG; if (flags & VALKEYMODULE_CONFIG_HIDDEN) new_flags |= HIDDEN_CONFIG; if (flags & VALKEYMODULE_CONFIG_PROTECTED) new_flags |= PROTECTED_CONFIG; if (flags & VALKEYMODULE_CONFIG_DENY_LOADING) new_flags |= DENY_LOADING_CONFIG; return new_flags; } unsigned int maskModuleNumericConfigFlags(unsigned int flags) { unsigned int new_flags = 0; if (flags & VALKEYMODULE_CONFIG_MEMORY) new_flags |= MEMORY_CONFIG; return new_flags; } unsigned int maskModuleEnumConfigFlags(unsigned int flags) { unsigned int new_flags = 0; if (flags & VALKEYMODULE_CONFIG_BITFLAGS) new_flags |= MULTI_ARG_CONFIG; return new_flags; } /* Create a string config that users can interact with via the server config file, * `CONFIG SET`, `CONFIG GET`, and `CONFIG REWRITE` commands. * * The actual config value is owned by the module, and the `getfn`, `setfn` and optional * `applyfn` callbacks that are provided to the server in order to access or manipulate the * value. The `getfn` callback retrieves the value from the module, while the `setfn` * callback provides a value to be stored into the module config. * The optional `applyfn` callback is called after a `CONFIG SET` command modified one or * more configs using the `setfn` callback and can be used to atomically apply a config * after several configs were changed together. * If there are multiple configs with `applyfn` callbacks set by a single `CONFIG SET` * command, they will be deduplicated if their `applyfn` function and `privdata` pointers * are identical, and the callback will only be run once. * Both the `setfn` and `applyfn` can return an error if the provided value is invalid or * cannot be used. * The config also declares a type for the value that is validated by the server and * provided to the module. The config system provides the following types: * * * String: Binary safe string data. * * Enum: One of a finite number of string tokens, provided during registration. * * Numeric: 64 bit signed integer, which also supports min and max values. * * Bool: Yes or no value. * * The `setfn` callback is expected to return VALKEYMODULE_OK when the value is successfully * applied. It can also return VALKEYMODULE_ERR if the value can't be applied, and the * *err pointer can be set with a ValkeyModuleString error message to provide to the client. * This ValkeyModuleString will be freed by the server after returning from the set callback. * * All configs are registered with a name, a type, a default value, private data that is made * available in the callbacks, as well as several flags that modify the behavior of the config. * The name must only contain alphanumeric characters or dashes. The supported flags are: * * * VALKEYMODULE_CONFIG_DEFAULT: The default flags for a config. This creates a config that can be modified after * startup. * * VALKEYMODULE_CONFIG_IMMUTABLE: This config can only be provided loading time. * * VALKEYMODULE_CONFIG_SENSITIVE: The value stored in this config is redacted from all logging. * * VALKEYMODULE_CONFIG_HIDDEN: The name is hidden from `CONFIG GET` with pattern matching. * * VALKEYMODULE_CONFIG_PROTECTED: This config will be only be modifiable based off the value of * enable-protected-configs. * * VALKEYMODULE_CONFIG_DENY_LOADING: This config is not modifiable while the server is loading data. * * VALKEYMODULE_CONFIG_MEMORY: For numeric configs, this config will convert data unit notations into their byte * equivalent. * * VALKEYMODULE_CONFIG_BITFLAGS: For enum configs, this config will allow multiple entries to be combined as bit * flags. * * Default values are used on startup to set the value if it is not provided via the config file * or command line. Default values are also used to compare to on a config rewrite. * * Notes: * * 1. On string config sets that the string passed to the set callback will be freed after execution and the module * must retain it. * 2. On string config gets the string will not be consumed and will be valid after execution. * * Example implementation: * * ValkeyModuleString *strval; * int adjustable = 1; * ValkeyModuleString *getStringConfigCommand(const char *name, void *privdata) { * return strval; * } * * int setStringConfigCommand(const char *name, ValkeyModuleString *new, void *privdata, ValkeyModuleString **err) { * if (adjustable) { * ValkeyModule_Free(strval); * ValkeyModule_RetainString(NULL, new); * strval = new; * return VALKEYMODULE_OK; * } * *err = ValkeyModule_CreateString(NULL, "Not adjustable.", 15); * return VALKEYMODULE_ERR; * } * ... * ValkeyModule_RegisterStringConfig(ctx, "string", NULL, VALKEYMODULE_CONFIG_DEFAULT, getStringConfigCommand, * setStringConfigCommand, NULL, NULL); * * If the registration fails, VALKEYMODULE_ERR is returned and one of the following * errno is set: * * EBUSY: Registering the Config outside of ValkeyModule_OnLoad. * * EINVAL: The provided flags are invalid for the registration or the name of the config contains invalid characters. * * EALREADY: The provided configuration name is already used. */ int VM_RegisterStringConfig(ValkeyModuleCtx *ctx, const char *name, const char *default_val, unsigned int flags, ValkeyModuleConfigGetStringFunc getfn, ValkeyModuleConfigSetStringFunc setfn, ValkeyModuleConfigApplyFunc applyfn, void *privdata) { ValkeyModule *module = ctx->module; if (moduleConfigValidityCheck(module, name, flags, NUMERIC_CONFIG)) { return VALKEYMODULE_ERR; } ModuleConfig *new_config = createModuleConfig(name, applyfn, privdata, module); new_config->get_fn.get_string = getfn; new_config->set_fn.set_string = setfn; listAddNodeTail(module->module_configs, new_config); flags = maskModuleConfigFlags(flags); addModuleStringConfig(module->name, name, flags, new_config, default_val ? sdsnew(default_val) : NULL); return VALKEYMODULE_OK; } /* Create a bool config that server clients can interact with via the * `CONFIG SET`, `CONFIG GET`, and `CONFIG REWRITE` commands. See * ValkeyModule_RegisterStringConfig for detailed information about configs. */ int VM_RegisterBoolConfig(ValkeyModuleCtx *ctx, const char *name, int default_val, unsigned int flags, ValkeyModuleConfigGetBoolFunc getfn, ValkeyModuleConfigSetBoolFunc setfn, ValkeyModuleConfigApplyFunc applyfn, void *privdata) { ValkeyModule *module = ctx->module; if (moduleConfigValidityCheck(module, name, flags, BOOL_CONFIG)) { return VALKEYMODULE_ERR; } ModuleConfig *new_config = createModuleConfig(name, applyfn, privdata, module); new_config->get_fn.get_bool = getfn; new_config->set_fn.set_bool = setfn; listAddNodeTail(module->module_configs, new_config); flags = maskModuleConfigFlags(flags); addModuleBoolConfig(module->name, name, flags, new_config, default_val); return VALKEYMODULE_OK; } /* * Create an enum config that server clients can interact with via the * `CONFIG SET`, `CONFIG GET`, and `CONFIG REWRITE` commands. * Enum configs are a set of string tokens to corresponding integer values, where * the string value is exposed to clients but the inter value is passed to the server * and the module. These values are defined in enum_values, an array * of null-terminated c strings, and int_vals, an array of enum values who has an * index partner in enum_values. * Example Implementation: * const char *enum_vals[3] = {"first", "second", "third"}; * const int int_vals[3] = {0, 2, 4}; * int enum_val = 0; * * int getEnumConfigCommand(const char *name, void *privdata) { * return enum_val; * } * * int setEnumConfigCommand(const char *name, int val, void *privdata, const char **err) { * enum_val = val; * return VALKEYMODULE_OK; * } * ... * ValkeyModule_RegisterEnumConfig(ctx, "enum", 0, VALKEYMODULE_CONFIG_DEFAULT, enum_vals, int_vals, 3, * getEnumConfigCommand, setEnumConfigCommand, NULL, NULL); * * Note that you can use VALKEYMODULE_CONFIG_BITFLAGS so that multiple enum string * can be combined into one integer as bit flags, in which case you may want to * sort your enums so that the preferred combinations are present first. * * See ValkeyModule_RegisterStringConfig for detailed general information about configs. */ int VM_RegisterEnumConfig(ValkeyModuleCtx *ctx, const char *name, int default_val, unsigned int flags, const char **enum_values, const int *int_values, int num_enum_vals, ValkeyModuleConfigGetEnumFunc getfn, ValkeyModuleConfigSetEnumFunc setfn, ValkeyModuleConfigApplyFunc applyfn, void *privdata) { ValkeyModule *module = ctx->module; if (moduleConfigValidityCheck(module, name, flags, ENUM_CONFIG)) { return VALKEYMODULE_ERR; } ModuleConfig *new_config = createModuleConfig(name, applyfn, privdata, module); new_config->get_fn.get_enum = getfn; new_config->set_fn.set_enum = setfn; configEnum *enum_vals = zmalloc((num_enum_vals + 1) * sizeof(configEnum)); for (int i = 0; i < num_enum_vals; i++) { enum_vals[i].name = zstrdup(enum_values[i]); enum_vals[i].val = int_values[i]; } enum_vals[num_enum_vals].name = NULL; enum_vals[num_enum_vals].val = 0; listAddNodeTail(module->module_configs, new_config); flags = maskModuleConfigFlags(flags) | maskModuleEnumConfigFlags(flags); addModuleEnumConfig(module->name, name, flags, new_config, default_val, enum_vals); return VALKEYMODULE_OK; } /* * Create an integer config that server clients can interact with via the * `CONFIG SET`, `CONFIG GET`, and `CONFIG REWRITE` commands. See * ValkeyModule_RegisterStringConfig for detailed information about configs. */ int VM_RegisterNumericConfig(ValkeyModuleCtx *ctx, const char *name, long long default_val, unsigned int flags, long long min, long long max, ValkeyModuleConfigGetNumericFunc getfn, ValkeyModuleConfigSetNumericFunc setfn, ValkeyModuleConfigApplyFunc applyfn, void *privdata) { ValkeyModule *module = ctx->module; if (moduleConfigValidityCheck(module, name, flags, NUMERIC_CONFIG)) { return VALKEYMODULE_ERR; } ModuleConfig *new_config = createModuleConfig(name, applyfn, privdata, module); new_config->get_fn.get_numeric = getfn; new_config->set_fn.set_numeric = setfn; listAddNodeTail(module->module_configs, new_config); unsigned int numeric_flags = maskModuleNumericConfigFlags(flags); flags = maskModuleConfigFlags(flags); addModuleNumericConfig(module->name, name, flags, new_config, default_val, numeric_flags, min, max); return VALKEYMODULE_OK; } /* Applies all pending configurations on the module load. This should be called * after all of the configurations have been registered for the module inside of ValkeyModule_OnLoad. * This will return VALKEYMODULE_ERR if it is called outside ValkeyModule_OnLoad. * This API needs to be called when configurations are provided in either `MODULE LOADEX` * or provided as startup arguments. */ int VM_LoadConfigs(ValkeyModuleCtx *ctx) { if (!ctx || !ctx->module || !ctx->module->onload) { return VALKEYMODULE_ERR; } ValkeyModule *module = ctx->module; /* Load configs from conf file or arguments from loadex */ if (loadModuleConfigs(module)) return VALKEYMODULE_ERR; return VALKEYMODULE_OK; } /* -------------------------------------------------------------------------- * ## RDB load/save API * -------------------------------------------------------------------------- */ #define VALKEYMODULE_RDB_STREAM_FILE 1 typedef struct ValkeyModuleRdbStream { int type; union { char *filename; } data; } ValkeyModuleRdbStream; /* Create a stream object to save/load RDB to/from a file. * * This function returns a pointer to ValkeyModuleRdbStream which is owned * by the caller. It requires a call to VM_RdbStreamFree() to free * the object. */ ValkeyModuleRdbStream *VM_RdbStreamCreateFromFile(const char *filename) { ValkeyModuleRdbStream *stream = zmalloc(sizeof(*stream)); stream->type = VALKEYMODULE_RDB_STREAM_FILE; stream->data.filename = zstrdup(filename); return stream; } /* Release an RDB stream object. */ void VM_RdbStreamFree(ValkeyModuleRdbStream *stream) { switch (stream->type) { case VALKEYMODULE_RDB_STREAM_FILE: zfree(stream->data.filename); break; default: serverAssert(0); break; } zfree(stream); } /* Load RDB file from the `stream`. Dataset will be cleared first and then RDB * file will be loaded. * * `flags` must be zero. This parameter is for future use. * * On success VALKEYMODULE_OK is returned, otherwise VALKEYMODULE_ERR is returned * and errno is set accordingly. * * Example: * * ValkeyModuleRdbStream *s = ValkeyModule_RdbStreamCreateFromFile("exp.rdb"); * ValkeyModule_RdbLoad(ctx, s, 0); * ValkeyModule_RdbStreamFree(s); */ int VM_RdbLoad(ValkeyModuleCtx *ctx, ValkeyModuleRdbStream *stream, int flags) { UNUSED(ctx); if (!stream || flags != 0) { errno = EINVAL; return VALKEYMODULE_ERR; } /* Not allowed on replicas. */ if (server.primary_host != NULL) { errno = ENOTSUP; return VALKEYMODULE_ERR; } /* Drop replicas if exist. */ disconnectReplicas(); freeReplicationBacklog(); /* Stop and kill existing AOF rewriting fork as it is saving outdated data, * we will re-enable it after the rdbLoad. Also killing it will prevent COW * memory issue. */ if (server.aof_state != AOF_OFF) stopAppendOnly(); /* Kill existing RDB fork as it is saving outdated data. Also killing it * will prevent COW memory issue. */ if (server.child_type == CHILD_TYPE_RDB) killRDBChild(); emptyData(-1, EMPTYDB_NO_FLAGS, NULL); /* rdbLoad() can go back to the networking and process network events. If * VM_RdbLoad() is called inside a command callback, we don't want to * process the current client. Otherwise, we may free the client or try to * process next message while we are already in the command callback. */ if (server.current_client) protectClient(server.current_client); serverAssert(stream->type == VALKEYMODULE_RDB_STREAM_FILE); int ret = rdbLoad(stream->data.filename, NULL, RDBFLAGS_NONE); if (server.current_client) unprotectClient(server.current_client); /* Here we need to decide whether to enable the AOF based on the aof_enabled, * since the previous stopAppendOnly sets aof_state to AOF_OFF. */ if (server.aof_enabled) startAppendOnly(); if (ret != RDB_OK) { errno = (ret == RDB_NOT_EXIST) ? ENOENT : EIO; return VALKEYMODULE_ERR; } errno = 0; return VALKEYMODULE_OK; } /* Save dataset to the RDB stream. * * `flags` must be zero. This parameter is for future use. * * On success VALKEYMODULE_OK is returned, otherwise VALKEYMODULE_ERR is returned * and errno is set accordingly. * * Example: * * ValkeyModuleRdbStream *s = ValkeyModule_RdbStreamCreateFromFile("exp.rdb"); * ValkeyModule_RdbSave(ctx, s, 0); * ValkeyModule_RdbStreamFree(s); */ int VM_RdbSave(ValkeyModuleCtx *ctx, ValkeyModuleRdbStream *stream, int flags) { UNUSED(ctx); if (!stream || flags != 0) { errno = EINVAL; return VALKEYMODULE_ERR; } serverAssert(stream->type == VALKEYMODULE_RDB_STREAM_FILE); if (rdbSaveToFile(stream->data.filename) != C_OK) { return VALKEYMODULE_ERR; } errno = 0; return VALKEYMODULE_OK; } /* Registers a new scripting engine in the server. * * - `module_ctx`: the module context object. * * - `engine_name`: the name of the scripting engine. This name will match * against the engine name specified in the script header using a shebang. * * - `engine_ctx`: engine specific context pointer. * * - `engine_methods`: the struct with the scripting engine callback functions * pointers. * * Returns VALKEYMODULE_OK if the engine is successfully registered, and * VALKEYMODULE_ERR in case some failure occurs. In case of a failure, an error * message is logged. */ int VM_RegisterScriptingEngine(ValkeyModuleCtx *module_ctx, const char *engine_name, ValkeyModuleScriptingEngineCtx *engine_ctx, ValkeyModuleScriptingEngineMethods *engine_methods) { serverLog(LL_DEBUG, "Registering a new scripting engine: %s", engine_name); if (engine_methods->version > VALKEYMODULE_SCRIPTING_ENGINE_ABI_VERSION) { serverLog(LL_WARNING, "The engine implementation version is greater " "than what this server supports. Server ABI " "Version: %lu, Engine ABI version: %lu", VALKEYMODULE_SCRIPTING_ENGINE_ABI_VERSION, (unsigned long)engine_methods->version); return VALKEYMODULE_ERR; } if (scriptingEngineManagerRegister(engine_name, module_ctx->module, engine_ctx, engine_methods) != C_OK) { return VALKEYMODULE_ERR; } return VALKEYMODULE_OK; } /* Removes the scripting engine from the server. * * `engine_name` is the name of the scripting engine. * * Returns VALKEYMODULE_OK. * */ int VM_UnregisterScriptingEngine(ValkeyModuleCtx *ctx, const char *engine_name) { UNUSED(ctx); if (scriptingEngineManagerUnregister(engine_name) != C_OK) { return VALKEYMODULE_ERR; } return VALKEYMODULE_OK; } /* MODULE command. * * MODULE LIST * MODULE LOAD [args...] * MODULE LOADEX [[CONFIG NAME VALUE] [CONFIG NAME VALUE]] [ARGS ...] * MODULE UNLOAD */ void moduleCommand(client *c) { char *subcmd = c->argv[1]->ptr; if (c->argc == 2 && !strcasecmp(subcmd, "help")) { const char *help[] = { "LIST", " Return a list of loaded modules.", "LOAD [ ...]", " Load a module library from , passing to it any optional arguments.", "LOADEX [[CONFIG NAME VALUE] [CONFIG NAME VALUE]] [ARGS ...]", " Load a module library from , while passing it module configurations and optional arguments.", "UNLOAD ", " Unload a module.", NULL}; addReplyHelp(c, help); } else if (!strcasecmp(subcmd, "load") && c->argc >= 3) { robj **argv = NULL; int argc = 0; if (c->argc > 3) { argc = c->argc - 3; argv = &c->argv[3]; } if (moduleLoad(c->argv[2]->ptr, (void **)argv, argc, 0) == C_OK) addReply(c, shared.ok); else addReplyError(c, "Error loading the extension. Please check the server logs."); } else if (!strcasecmp(subcmd, "loadex") && c->argc >= 3) { robj **argv = NULL; int argc = 0; if (c->argc > 3) { argc = c->argc - 3; argv = &c->argv[3]; } /* If this is a loadex command we want to populate server.module_configs_queue with * sds NAME VALUE pairs. We also want to increment argv to just after ARGS, if supplied. */ if (parseLoadexArguments((ValkeyModuleString ***)&argv, &argc) == VALKEYMODULE_OK && moduleLoad(c->argv[2]->ptr, (void **)argv, argc, 1) == C_OK) addReply(c, shared.ok); else { dictEmpty(server.module_configs_queue, NULL); addReplyError(c, "Error loading the extension. Please check the server logs."); } } else if (!strcasecmp(subcmd, "unload") && c->argc == 3) { const char *errmsg = NULL; if (moduleUnload(c->argv[2]->ptr, &errmsg) == C_OK) addReply(c, shared.ok); else { if (errmsg == NULL) errmsg = "operation not possible."; addReplyErrorFormat(c, "Error unloading module: %s", errmsg); serverLog(LL_WARNING, "Error unloading module %s: %s", (sds)c->argv[2]->ptr, errmsg); } } else if (!strcasecmp(subcmd, "list") && c->argc == 2) { addReplyLoadedModules(c); } else { addReplySubcommandSyntaxError(c); return; } } /* Return the number of registered modules. */ size_t moduleCount(void) { return dictSize(modules); } /* -------------------------------------------------------------------------- * ## Key eviction API * -------------------------------------------------------------------------- */ /* Set the key last access time for LRU based eviction. not relevant if the * servers's maxmemory policy is LFU based. Value is idle time in milliseconds. * returns VALKEYMODULE_OK if the LRU was updated, VALKEYMODULE_ERR otherwise. */ int VM_SetLRU(ValkeyModuleKey *key, mstime_t lru_idle) { if (!key->value) return VALKEYMODULE_ERR; if (objectSetLRUOrLFU(key->value, -1, lru_idle, lru_idle >= 0 ? LRU_CLOCK() : 0, 1)) return VALKEYMODULE_OK; return VALKEYMODULE_ERR; } /* Gets the key last access time. * Value is idletime in milliseconds or -1 if the server's eviction policy is * LFU based. * returns VALKEYMODULE_OK if when key is valid. */ int VM_GetLRU(ValkeyModuleKey *key, mstime_t *lru_idle) { *lru_idle = -1; if (!key->value) return VALKEYMODULE_ERR; if (server.maxmemory_policy & MAXMEMORY_FLAG_LFU) return VALKEYMODULE_OK; *lru_idle = estimateObjectIdleTime(key->value); return VALKEYMODULE_OK; } /* Set the key access frequency. only relevant if the server's maxmemory policy * is LFU based. * The frequency is a logarithmic counter that provides an indication of * the access frequencyonly (must be <= 255). * returns VALKEYMODULE_OK if the LFU was updated, VALKEYMODULE_ERR otherwise. */ int VM_SetLFU(ValkeyModuleKey *key, long long lfu_freq) { if (!key->value) return VALKEYMODULE_ERR; if (objectSetLRUOrLFU(key->value, lfu_freq, -1, 0, 1)) return VALKEYMODULE_OK; return VALKEYMODULE_ERR; } /* Gets the key access frequency or -1 if the server's eviction policy is not * LFU based. * returns VALKEYMODULE_OK if when key is valid. */ int VM_GetLFU(ValkeyModuleKey *key, long long *lfu_freq) { *lfu_freq = -1; if (!key->value) return VALKEYMODULE_ERR; if (server.maxmemory_policy & MAXMEMORY_FLAG_LFU) *lfu_freq = LFUDecrAndReturn(key->value); return VALKEYMODULE_OK; } /* -------------------------------------------------------------------------- * ## Miscellaneous APIs * -------------------------------------------------------------------------- */ /** * Returns the full module options flags mask, using the return value * the module can check if a certain set of module options are supported * by the server version in use. * Example: * * int supportedFlags = VM_GetModuleOptionsAll(); * if (supportedFlags & VALKEYMODULE_OPTIONS_ALLOW_NESTED_KEYSPACE_NOTIFICATIONS) { * // VALKEYMODULE_OPTIONS_ALLOW_NESTED_KEYSPACE_NOTIFICATIONS is supported * } else{ * // VALKEYMODULE_OPTIONS_ALLOW_NESTED_KEYSPACE_NOTIFICATIONS is not supported * } */ int VM_GetModuleOptionsAll(void) { return _VALKEYMODULE_OPTIONS_FLAGS_NEXT - 1; } /** * Returns the full ContextFlags mask, using the return value * the module can check if a certain set of flags are supported * by the server version in use. * Example: * * int supportedFlags = VM_GetContextFlagsAll(); * if (supportedFlags & VALKEYMODULE_CTX_FLAGS_MULTI) { * // VALKEYMODULE_CTX_FLAGS_MULTI is supported * } else{ * // VALKEYMODULE_CTX_FLAGS_MULTI is not supported * } */ int VM_GetContextFlagsAll(void) { return _VALKEYMODULE_CTX_FLAGS_NEXT - 1; } /** * Returns the full KeyspaceNotification mask, using the return value * the module can check if a certain set of flags are supported * by the server version in use. * Example: * * int supportedFlags = VM_GetKeyspaceNotificationFlagsAll(); * if (supportedFlags & VALKEYMODULE_NOTIFY_LOADED) { * // VALKEYMODULE_NOTIFY_LOADED is supported * } else{ * // VALKEYMODULE_NOTIFY_LOADED is not supported * } */ int VM_GetKeyspaceNotificationFlagsAll(void) { return _VALKEYMODULE_NOTIFY_NEXT - 1; } /** * Return the server version in format of 0x00MMmmpp. * Example for 6.0.7 the return value will be 0x00060007. */ int VM_GetServerVersion(void) { return VALKEY_VERSION_NUM; } /** * Return the current server runtime value of VALKEYMODULE_TYPE_METHOD_VERSION. * You can use that when calling VM_CreateDataType to know which fields of * ValkeyModuleTypeMethods are gonna be supported and which will be ignored. */ int VM_GetTypeMethodVersion(void) { return VALKEYMODULE_TYPE_METHOD_VERSION; } /* Replace the value assigned to a module type. * * The key must be open for writing, have an existing value, and have a moduleType * that matches the one specified by the caller. * * Unlike VM_ModuleTypeSetValue() which will free the old value, this function * simply swaps the old value with the new value. * * The function returns VALKEYMODULE_OK on success, VALKEYMODULE_ERR on errors * such as: * * 1. Key is not opened for writing. * 2. Key is not a module data type key. * 3. Key is a module datatype other than 'mt'. * * If old_value is non-NULL, the old value is returned by reference. */ int VM_ModuleTypeReplaceValue(ValkeyModuleKey *key, moduleType *mt, void *new_value, void **old_value) { if (!(key->mode & VALKEYMODULE_WRITE) || key->iter) return VALKEYMODULE_ERR; if (!key->value || key->value->type != OBJ_MODULE) return VALKEYMODULE_ERR; moduleValue *mv = key->value->ptr; if (mv->type != mt) return VALKEYMODULE_ERR; if (old_value) *old_value = mv->value; mv->value = new_value; return VALKEYMODULE_OK; } /* For a specified command, parse its arguments and return an array that * contains the indexes of all key name arguments. This function is * essentially a more efficient way to do `COMMAND GETKEYS`. * * The out_flags argument is optional, and can be set to NULL. * When provided it is filled with VALKEYMODULE_CMD_KEY_ flags in matching * indexes with the key indexes of the returned array. * * A NULL return value indicates the specified command has no keys, or * an error condition. Error conditions are indicated by setting errno * as follows: * * * ENOENT: Specified command does not exist. * * EINVAL: Invalid command arity specified. * * NOTE: The returned array is not a Module object so it does not * get automatically freed even when auto-memory is used. The caller * must explicitly call VM_Free() to free it, same as the out_flags pointer if * used. */ int *VM_GetCommandKeysWithFlags(ValkeyModuleCtx *ctx, ValkeyModuleString **argv, int argc, int *num_keys, int **out_flags) { UNUSED(ctx); struct serverCommand *cmd; int *res = NULL; /* Find command */ if ((cmd = lookupCommand(argv, argc)) == NULL) { errno = ENOENT; return NULL; } /* Bail out if command has no keys */ if (!doesCommandHaveKeys(cmd)) { errno = 0; return NULL; } if ((cmd->arity > 0 && cmd->arity != argc) || (argc < -cmd->arity)) { errno = EINVAL; return NULL; } getKeysResult result; initGetKeysResult(&result); getKeysFromCommand(cmd, argv, argc, &result); *num_keys = result.numkeys; if (!result.numkeys) { errno = 0; getKeysFreeResult(&result); return NULL; } /* The return value here expects an array of key positions */ unsigned long int size = sizeof(int) * result.numkeys; res = zmalloc(size); if (out_flags) *out_flags = zmalloc(size); for (int i = 0; i < result.numkeys; i++) { res[i] = result.keys[i].pos; if (out_flags) (*out_flags)[i] = moduleConvertKeySpecsFlags(result.keys[i].flags, 0); } return res; } /* Identical to VM_GetCommandKeysWithFlags when flags are not needed. */ int *VM_GetCommandKeys(ValkeyModuleCtx *ctx, ValkeyModuleString **argv, int argc, int *num_keys) { return VM_GetCommandKeysWithFlags(ctx, argv, argc, num_keys, NULL); } /* Return the name of the command currently running */ const char *VM_GetCurrentCommandName(ValkeyModuleCtx *ctx) { if (!ctx || !ctx->client || !ctx->client->cmd) return NULL; return (const char *)ctx->client->cmd->fullname; } /* -------------------------------------------------------------------------- * ## Defrag API * -------------------------------------------------------------------------- */ /* The defrag context, used to manage state during calls to the data type * defrag callback. */ struct ValkeyModuleDefragCtx { monotime endtime; unsigned long *cursor; struct serverObject *key; /* Optional name of key processed, NULL when unknown. */ int dbid; /* The dbid of the key being processed, -1 when unknown. */ }; /* Register a defrag callback for global data, i.e. anything that the module * may allocate that is not tied to a specific data type. */ int VM_RegisterDefragFunc(ValkeyModuleCtx *ctx, ValkeyModuleDefragFunc cb) { ctx->module->defrag_cb = cb; return VALKEYMODULE_OK; } /* When the data type defrag callback iterates complex structures, this * function should be called periodically. A zero (false) return * indicates the callback may continue its work. A non-zero value (true) * indicates it should stop. * * When stopped, the callback may use VM_DefragCursorSet() to store its * position so it can later use VM_DefragCursorGet() to resume defragging. * * When stopped and more work is left to be done, the callback should * return 1. Otherwise, it should return 0. * * NOTE: Modules should consider the frequency in which this function is called, * so it generally makes sense to do small batches of work in between calls. */ int VM_DefragShouldStop(ValkeyModuleDefragCtx *ctx) { return (ctx->endtime != 0 && ctx->endtime <= getMonotonicUs()); } /* Store an arbitrary cursor value for future re-use. * * This should only be called if VM_DefragShouldStop() has returned a non-zero * value and the defrag callback is about to exit without fully iterating its * data type. * * This behavior is reserved to cases where late defrag is performed. Late * defrag is selected for keys that implement the `free_effort` callback and * return a `free_effort` value that is larger than the defrag * 'active-defrag-max-scan-fields' configuration directive. * * Smaller keys, keys that do not implement `free_effort` or the global * defrag callback are not called in late-defrag mode. In those cases, a * call to this function will return VALKEYMODULE_ERR. * * The cursor may be used by the module to represent some progress into the * module's data type. Modules may also store additional cursor-related * information locally and use the cursor as a flag that indicates when * traversal of a new key begins. This is possible because the API makes * a guarantee that concurrent defragmentation of multiple keys will * not be performed. */ int VM_DefragCursorSet(ValkeyModuleDefragCtx *ctx, unsigned long cursor) { if (!ctx->cursor) return VALKEYMODULE_ERR; *ctx->cursor = cursor; return VALKEYMODULE_OK; } /* Fetch a cursor value that has been previously stored using VM_DefragCursorSet(). * * If not called for a late defrag operation, VALKEYMODULE_ERR will be returned and * the cursor should be ignored. See VM_DefragCursorSet() for more details on * defrag cursors. */ int VM_DefragCursorGet(ValkeyModuleDefragCtx *ctx, unsigned long *cursor) { if (!ctx->cursor) return VALKEYMODULE_ERR; *cursor = *ctx->cursor; return VALKEYMODULE_OK; } /* Defrag a memory allocation previously allocated by VM_Alloc, VM_Calloc, etc. * The defragmentation process involves allocating a new memory block and copying * the contents to it, like realloc(). * * If defragmentation was not necessary, NULL is returned and the operation has * no other effect. * * If a non-NULL value is returned, the caller should use the new pointer instead * of the old one and update any reference to the old pointer, which must not * be used again. */ void *VM_DefragAlloc(ValkeyModuleDefragCtx *ctx, void *ptr) { UNUSED(ctx); return activeDefragAlloc(ptr); } /* Defrag a ValkeyModuleString previously allocated by VM_Alloc, VM_Calloc, etc. * See VM_DefragAlloc() for more information on how the defragmentation process * works. * * NOTE: It is only possible to defrag strings that have a single reference. * Typically this means strings retained with VM_RetainString or VM_HoldString * may not be defragmentable. One exception is command argvs which, if retained * by the module, will end up with a single reference (because the reference * on the server side is dropped as soon as the command callback returns). */ ValkeyModuleString *VM_DefragValkeyModuleString(ValkeyModuleDefragCtx *ctx, ValkeyModuleString *str) { UNUSED(ctx); return activeDefragStringOb(str); } /* Perform a late defrag of a module datatype key. * * Returns a zero value (and initializes the cursor) if no more needs to be done, * or a non-zero value otherwise. */ int moduleLateDefrag(robj *key, robj *value, unsigned long *cursor, monotime endtime, int dbid) { moduleValue *mv = value->ptr; moduleType *mt = mv->type; ValkeyModuleDefragCtx defrag_ctx = {endtime, cursor, key, dbid}; /* Invoke callback. Note that the callback may be missing if the key has been * replaced with a different type since our last visit. */ int ret = 0; if (mt->defrag) ret = mt->defrag(&defrag_ctx, key, &mv->value); if (!ret) { *cursor = 0; /* No more work to do */ return 0; } return 1; } /* Attempt to defrag a module data type value. Depending on complexity, * the operation may happen immediately or be scheduled for later. * * Returns 1 if the operation has been completed or 0 if it needs to * be scheduled for late defrag. */ int moduleDefragValue(robj *key, robj *value, int dbid) { moduleValue *mv = value->ptr; moduleType *mt = mv->type; /* Try to defrag moduleValue itself regardless of whether or not * defrag callbacks are provided. */ moduleValue *newmv = activeDefragAlloc(mv); if (newmv) { value->ptr = mv = newmv; } if (!mt->defrag) return 1; /* Use free_effort to determine complexity of module value, and if * necessary schedule it for defragLater instead of quick immediate * defrag. */ size_t effort = moduleGetFreeEffort(key, value, dbid); if (!effort) effort = SIZE_MAX; if (effort > server.active_defrag_max_scan_fields) { return 0; /* Defrag later */ } ValkeyModuleDefragCtx defrag_ctx = {0, NULL, key, dbid}; mt->defrag(&defrag_ctx, key, &mv->value); return 1; } /* Call registered module API defrag functions */ void moduleDefragGlobals(void) { dictIterator *di = dictGetIterator(modules); dictEntry *de; while ((de = dictNext(di)) != NULL) { struct ValkeyModule *module = dictGetVal(de); if (!module->defrag_cb) continue; ValkeyModuleDefragCtx defrag_ctx = {0, NULL, NULL, -1}; module->defrag_cb(&defrag_ctx); } dictReleaseIterator(di); } /* Returns the name of the key currently being processed. * There is no guarantee that the key name is always available, so this may return NULL. */ const ValkeyModuleString *VM_GetKeyNameFromDefragCtx(ValkeyModuleDefragCtx *ctx) { return ctx->key; } /* Returns the database id of the key currently being processed. * There is no guarantee that this info is always available, so this may return -1. */ int VM_GetDbIdFromDefragCtx(ValkeyModuleDefragCtx *ctx) { return ctx->dbid; } /* Register all the APIs we export. Keep this function at the end of the * file so that's easy to seek it to add new entries. */ void moduleRegisterCoreAPI(void) { server.moduleapi = dictCreate(&moduleAPIDictType); server.sharedapi = dictCreate(&moduleAPIDictType); REGISTER_API(Alloc); REGISTER_API(TryAlloc); REGISTER_API(Calloc); REGISTER_API(TryCalloc); REGISTER_API(Realloc); REGISTER_API(TryRealloc); REGISTER_API(Free); REGISTER_API(Strdup); REGISTER_API(CreateCommand); REGISTER_API(GetCommand); REGISTER_API(CreateSubcommand); REGISTER_API(SetCommandInfo); REGISTER_API(SetCommandACLCategories); REGISTER_API(AddACLCategory); REGISTER_API(SetModuleAttribs); REGISTER_API(IsModuleNameBusy); REGISTER_API(WrongArity); REGISTER_API(UpdateRuntimeArgs); REGISTER_API(ReplyWithLongLong); REGISTER_API(ReplyWithError); REGISTER_API(ReplyWithErrorFormat); REGISTER_API(ReplyWithSimpleString); REGISTER_API(ReplyWithArray); REGISTER_API(ReplyWithMap); REGISTER_API(ReplyWithSet); REGISTER_API(ReplyWithAttribute); REGISTER_API(ReplyWithNullArray); REGISTER_API(ReplyWithEmptyArray); REGISTER_API(ReplySetArrayLength); REGISTER_API(ReplySetMapLength); REGISTER_API(ReplySetSetLength); REGISTER_API(ReplySetAttributeLength); REGISTER_API(ReplyWithString); REGISTER_API(ReplyWithEmptyString); REGISTER_API(ReplyWithVerbatimString); REGISTER_API(ReplyWithVerbatimStringType); REGISTER_API(ReplyWithStringBuffer); REGISTER_API(ReplyWithCString); REGISTER_API(ReplyWithNull); REGISTER_API(ReplyWithBool); REGISTER_API(ReplyWithCallReply); REGISTER_API(ReplyWithDouble); REGISTER_API(ReplyWithBigNumber); REGISTER_API(ReplyWithLongDouble); REGISTER_API(GetSelectedDb); REGISTER_API(SelectDb); REGISTER_API(KeyExists); REGISTER_API(OpenKey); REGISTER_API(GetOpenKeyModesAll); REGISTER_API(CloseKey); REGISTER_API(KeyType); REGISTER_API(ValueLength); REGISTER_API(ListPush); REGISTER_API(ListPop); REGISTER_API(ListGet); REGISTER_API(ListSet); REGISTER_API(ListInsert); REGISTER_API(ListDelete); REGISTER_API(StringToLongLong); REGISTER_API(StringToULongLong); REGISTER_API(StringToDouble); REGISTER_API(StringToLongDouble); REGISTER_API(StringToStreamID); REGISTER_API(Call); REGISTER_API(CallReplyProto); REGISTER_API(FreeCallReply); REGISTER_API(CallReplyInteger); REGISTER_API(CallReplyDouble); REGISTER_API(CallReplyBigNumber); REGISTER_API(CallReplyVerbatim); REGISTER_API(CallReplyBool); REGISTER_API(CallReplySetElement); REGISTER_API(CallReplyMapElement); REGISTER_API(CallReplyAttributeElement); REGISTER_API(CallReplyPromiseSetUnblockHandler); REGISTER_API(CallReplyPromiseAbort); REGISTER_API(CallReplyAttribute); REGISTER_API(CallReplyType); REGISTER_API(CallReplyLength); REGISTER_API(CallReplyArrayElement); REGISTER_API(CallReplyStringPtr); REGISTER_API(CreateStringFromCallReply); REGISTER_API(CreateString); REGISTER_API(CreateStringFromLongLong); REGISTER_API(CreateStringFromULongLong); REGISTER_API(CreateStringFromDouble); REGISTER_API(CreateStringFromLongDouble); REGISTER_API(CreateStringFromString); REGISTER_API(CreateStringFromStreamID); REGISTER_API(CreateStringPrintf); REGISTER_API(FreeString); REGISTER_API(StringPtrLen); REGISTER_API(AutoMemory); REGISTER_API(Replicate); REGISTER_API(ReplicateVerbatim); REGISTER_API(DeleteKey); REGISTER_API(UnlinkKey); REGISTER_API(StringSet); REGISTER_API(StringDMA); REGISTER_API(StringTruncate); REGISTER_API(SetExpire); REGISTER_API(GetExpire); REGISTER_API(SetAbsExpire); REGISTER_API(GetAbsExpire); REGISTER_API(ResetDataset); REGISTER_API(DbSize); REGISTER_API(RandomKey); REGISTER_API(ZsetAdd); REGISTER_API(ZsetIncrby); REGISTER_API(ZsetScore); REGISTER_API(ZsetRem); REGISTER_API(ZsetRangeStop); REGISTER_API(ZsetFirstInScoreRange); REGISTER_API(ZsetLastInScoreRange); REGISTER_API(ZsetFirstInLexRange); REGISTER_API(ZsetLastInLexRange); REGISTER_API(ZsetRangeCurrentElement); REGISTER_API(ZsetRangeNext); REGISTER_API(ZsetRangePrev); REGISTER_API(ZsetRangeEndReached); REGISTER_API(HashSet); REGISTER_API(HashGet); REGISTER_API(StreamAdd); REGISTER_API(StreamDelete); REGISTER_API(StreamIteratorStart); REGISTER_API(StreamIteratorStop); REGISTER_API(StreamIteratorNextID); REGISTER_API(StreamIteratorNextField); REGISTER_API(StreamIteratorDelete); REGISTER_API(StreamTrimByLength); REGISTER_API(StreamTrimByID); REGISTER_API(IsKeysPositionRequest); REGISTER_API(KeyAtPos); REGISTER_API(KeyAtPosWithFlags); REGISTER_API(IsChannelsPositionRequest); REGISTER_API(ChannelAtPosWithFlags); REGISTER_API(GetClientId); REGISTER_API(GetClientUserNameById); REGISTER_API(GetContextFlags); REGISTER_API(AvoidReplicaTraffic); REGISTER_API(PoolAlloc); REGISTER_API(CreateDataType); REGISTER_API(ModuleTypeSetValue); REGISTER_API(ModuleTypeReplaceValue); REGISTER_API(ModuleTypeGetType); REGISTER_API(ModuleTypeGetValue); REGISTER_API(IsIOError); REGISTER_API(SetModuleOptions); REGISTER_API(SignalModifiedKey); REGISTER_API(SaveUnsigned); REGISTER_API(LoadUnsigned); REGISTER_API(SaveSigned); REGISTER_API(LoadSigned); REGISTER_API(SaveString); REGISTER_API(SaveStringBuffer); REGISTER_API(LoadString); REGISTER_API(LoadStringBuffer); REGISTER_API(SaveDouble); REGISTER_API(LoadDouble); REGISTER_API(SaveFloat); REGISTER_API(LoadFloat); REGISTER_API(SaveLongDouble); REGISTER_API(LoadLongDouble); REGISTER_API(SaveDataTypeToString); REGISTER_API(LoadDataTypeFromString); REGISTER_API(LoadDataTypeFromStringEncver); REGISTER_API(EmitAOF); REGISTER_API(Log); REGISTER_API(LogIOError); REGISTER_API(_Assert); REGISTER_API(LatencyAddSample); REGISTER_API(StringAppendBuffer); REGISTER_API(TrimStringAllocation); REGISTER_API(RetainString); REGISTER_API(HoldString); REGISTER_API(StringCompare); REGISTER_API(GetContextFromIO); REGISTER_API(GetKeyNameFromIO); REGISTER_API(GetKeyNameFromModuleKey); REGISTER_API(GetDbIdFromModuleKey); REGISTER_API(GetDbIdFromIO); REGISTER_API(GetKeyNameFromOptCtx); REGISTER_API(GetToKeyNameFromOptCtx); REGISTER_API(GetDbIdFromOptCtx); REGISTER_API(GetToDbIdFromOptCtx); REGISTER_API(GetKeyNameFromDefragCtx); REGISTER_API(GetDbIdFromDefragCtx); REGISTER_API(GetKeyNameFromDigest); REGISTER_API(GetDbIdFromDigest); REGISTER_API(BlockClient); REGISTER_API(BlockClientGetPrivateData); REGISTER_API(BlockClientSetPrivateData); REGISTER_API(BlockClientOnAuth); REGISTER_API(UnblockClient); REGISTER_API(IsBlockedReplyRequest); REGISTER_API(IsBlockedTimeoutRequest); REGISTER_API(GetBlockedClientPrivateData); REGISTER_API(AbortBlock); REGISTER_API(Milliseconds); REGISTER_API(MonotonicMicroseconds); REGISTER_API(Microseconds); REGISTER_API(CachedMicroseconds); REGISTER_API(BlockedClientMeasureTimeStart); REGISTER_API(BlockedClientMeasureTimeEnd); REGISTER_API(GetThreadSafeContext); REGISTER_API(GetDetachedThreadSafeContext); REGISTER_API(FreeThreadSafeContext); REGISTER_API(ThreadSafeContextLock); REGISTER_API(ThreadSafeContextTryLock); REGISTER_API(ThreadSafeContextUnlock); REGISTER_API(DigestAddStringBuffer); REGISTER_API(DigestAddLongLong); REGISTER_API(DigestEndSequence); REGISTER_API(NotifyKeyspaceEvent); REGISTER_API(GetNotifyKeyspaceEvents); REGISTER_API(SubscribeToKeyspaceEvents); REGISTER_API(AddPostNotificationJob); REGISTER_API(RegisterClusterMessageReceiver); REGISTER_API(SendClusterMessage); REGISTER_API(GetClusterNodeInfo); REGISTER_API(GetClusterNodeInfoForClient); REGISTER_API(GetClusterNodesList); REGISTER_API(FreeClusterNodesList); REGISTER_API(CreateTimer); REGISTER_API(StopTimer); REGISTER_API(GetTimerInfo); REGISTER_API(GetMyClusterID); REGISTER_API(GetClusterSize); REGISTER_API(GetRandomBytes); REGISTER_API(GetRandomHexChars); REGISTER_API(BlockedClientDisconnected); REGISTER_API(SetDisconnectCallback); REGISTER_API(GetBlockedClientHandle); REGISTER_API(SetClusterFlags); REGISTER_API(ClusterKeySlot); REGISTER_API(ClusterCanonicalKeyNameInSlot); REGISTER_API(CreateDict); REGISTER_API(FreeDict); REGISTER_API(DictSize); REGISTER_API(DictSetC); REGISTER_API(DictReplaceC); REGISTER_API(DictSet); REGISTER_API(DictReplace); REGISTER_API(DictGetC); REGISTER_API(DictGet); REGISTER_API(DictDelC); REGISTER_API(DictDel); REGISTER_API(DictIteratorStartC); REGISTER_API(DictIteratorStart); REGISTER_API(DictIteratorStop); REGISTER_API(DictIteratorReseekC); REGISTER_API(DictIteratorReseek); REGISTER_API(DictNextC); REGISTER_API(DictPrevC); REGISTER_API(DictNext); REGISTER_API(DictPrev); REGISTER_API(DictCompareC); REGISTER_API(DictCompare); REGISTER_API(ExportSharedAPI); REGISTER_API(GetSharedAPI); REGISTER_API(RegisterCommandFilter); REGISTER_API(UnregisterCommandFilter); REGISTER_API(CommandFilterArgsCount); REGISTER_API(CommandFilterArgGet); REGISTER_API(CommandFilterArgInsert); REGISTER_API(CommandFilterArgReplace); REGISTER_API(CommandFilterArgDelete); REGISTER_API(CommandFilterGetClientId); REGISTER_API(Fork); REGISTER_API(SendChildHeartbeat); REGISTER_API(ExitFromChild); REGISTER_API(KillForkChild); REGISTER_API(RegisterInfoFunc); REGISTER_API(InfoAddSection); REGISTER_API(InfoBeginDictField); REGISTER_API(InfoEndDictField); REGISTER_API(InfoAddFieldString); REGISTER_API(InfoAddFieldCString); REGISTER_API(InfoAddFieldDouble); REGISTER_API(InfoAddFieldLongLong); REGISTER_API(InfoAddFieldULongLong); REGISTER_API(GetServerInfo); REGISTER_API(FreeServerInfo); REGISTER_API(ServerInfoGetField); REGISTER_API(ServerInfoGetFieldC); REGISTER_API(ServerInfoGetFieldSigned); REGISTER_API(ServerInfoGetFieldUnsigned); REGISTER_API(ServerInfoGetFieldDouble); REGISTER_API(GetClientInfoById); REGISTER_API(GetClientNameById); REGISTER_API(SetClientNameById); REGISTER_API(PublishMessage); REGISTER_API(PublishMessageShard); REGISTER_API(SubscribeToServerEvent); REGISTER_API(SetLRU); REGISTER_API(GetLRU); REGISTER_API(SetLFU); REGISTER_API(GetLFU); REGISTER_API(BlockClientOnKeys); REGISTER_API(BlockClientOnKeysWithFlags); REGISTER_API(SignalKeyAsReady); REGISTER_API(GetBlockedClientReadyKey); REGISTER_API(GetUsedMemoryRatio); REGISTER_API(MallocSize); REGISTER_API(MallocUsableSize); REGISTER_API(MallocSizeString); REGISTER_API(MallocSizeDict); REGISTER_API(ScanCursorCreate); REGISTER_API(ScanCursorDestroy); REGISTER_API(ScanCursorRestart); REGISTER_API(Scan); REGISTER_API(ScanKey); REGISTER_API(CreateModuleUser); REGISTER_API(SetContextUser); REGISTER_API(SetModuleUserACL); REGISTER_API(SetModuleUserACLString); REGISTER_API(GetModuleUserACLString); REGISTER_API(GetCurrentUserName); REGISTER_API(GetModuleUserFromUserName); REGISTER_API(ACLCheckCommandPermissions); REGISTER_API(ACLCheckKeyPermissions); REGISTER_API(ACLCheckChannelPermissions); REGISTER_API(ACLAddLogEntry); REGISTER_API(ACLAddLogEntryByUserName); REGISTER_API(FreeModuleUser); REGISTER_API(DeauthenticateAndCloseClient); REGISTER_API(AuthenticateClientWithACLUser); REGISTER_API(AuthenticateClientWithUser); REGISTER_API(GetContextFlagsAll); REGISTER_API(GetModuleOptionsAll); REGISTER_API(GetKeyspaceNotificationFlagsAll); REGISTER_API(IsSubEventSupported); REGISTER_API(GetServerVersion); REGISTER_API(GetClientCertificate); REGISTER_API(RedactClientCommandArgument); REGISTER_API(GetCommandKeys); REGISTER_API(GetCommandKeysWithFlags); REGISTER_API(GetCurrentCommandName); REGISTER_API(GetTypeMethodVersion); REGISTER_API(RegisterDefragFunc); REGISTER_API(DefragAlloc); REGISTER_API(DefragValkeyModuleString); REGISTER_API(DefragShouldStop); REGISTER_API(DefragCursorSet); REGISTER_API(DefragCursorGet); REGISTER_API(EventLoopAdd); REGISTER_API(EventLoopDel); REGISTER_API(EventLoopAddOneShot); REGISTER_API(Yield); REGISTER_API(RegisterBoolConfig); REGISTER_API(RegisterNumericConfig); REGISTER_API(RegisterStringConfig); REGISTER_API(RegisterEnumConfig); REGISTER_API(LoadConfigs); REGISTER_API(RegisterAuthCallback); REGISTER_API(RdbStreamCreateFromFile); REGISTER_API(RdbStreamFree); REGISTER_API(RdbLoad); REGISTER_API(RdbSave); REGISTER_API(RegisterScriptingEngine); REGISTER_API(UnregisterScriptingEngine); }