diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2582c53a4..2e1e7865c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -58,3 +58,14 @@ jobs: run: | yum -y install gcc make make REDIS_CFLAGS='-Werror' + + build-freebsd: + runs-on: macos-latest + steps: + - uses: actions/checkout@v2 + - name: make + uses: vmactions/freebsd-vm@v0.1.0 + with: + usesh: true + prepare: pkg install -y gmake + run: gmake diff --git a/.github/workflows/daily.yml b/.github/workflows/daily.yml index 028c44e0c..8fb23bac4 100644 --- a/.github/workflows/daily.yml +++ b/.github/workflows/daily.yml @@ -99,6 +99,23 @@ jobs: ./runtest-cluster --tls ./runtest-cluster + test-ubuntu-io-threads: + runs-on: ubuntu-latest + if: github.repository == 'redis/redis' + timeout-minutes: 14400 + steps: + - uses: actions/checkout@v2 + - name: make + run: | + make + - name: test + run: | + sudo apt-get install tcl8.5 tcl-tls + ./runtest --config io-threads 4 --config io-threads-do-reads yes --accurate --verbose --tags network + - name: cluster tests + run: | + ./runtest-cluster --config io-threads 4 --config io-threads-do-reads yes + test-valgrind: runs-on: ubuntu-latest if: github.repository == 'redis/redis' @@ -186,3 +203,20 @@ jobs: - name: cluster tests run: ./runtest-cluster + test-freebsd: + runs-on: macos-latest + if: github.repository == 'redis/redis' + timeout-minutes: 14400 + steps: + - uses: actions/checkout@v2 + - name: test + uses: vmactions/freebsd-vm@v0.1.0 + with: + usesh: true + prepare: pkg install -y gmake lang/tcl85 + run: | + gmake + ./runtest --accurate --verbose --no-latency + MAKE=gmake ./runtest-moduleapi --verbose + ./runtest-sentinel + ./runtest-cluster diff --git a/00-RELEASENOTES b/00-RELEASENOTES index 3434d4838..f52a22ed9 100644 --- a/00-RELEASENOTES +++ b/00-RELEASENOTES @@ -1,3 +1,53 @@ +Redis 6.2 RC3 Released Tue Feb 1 14:00:00 IST 2021 +================================================================================ + +Upgrade urgency LOW: This is the third Release Candidate of Redis 6.2. + +Here is a comprehensive list of changes in this release compared to 6.2 RC2, +each one includes the PR number that added it, so you can get more details +at https://github.com/redis/redis/pull/ + +New commands / args: +* Add HRANDFIELD and ZRANDMEMBER commands (#8297) +* Add FAILOVER command (#8315) +* Add GETEX, GETDEL commands (#8327) +* Add PXAT/EXAT arguments to SET command (#8327) +* Add SYNC arg to FLUSHALL and FLUSHDB, and ASYNC/SYNC arg to SCRIPT FLUSH (#8258) + +Sentinel: +* Add hostname support to Sentinel (#8282) +* Prevent file descriptors from leaking into Sentinel scripts (#8242) +* Fix config file line order dependency and config rewrite sequence (#8271) + +New configuration options: +* Add set-proc-title config option to disable changes to the process title (#3623) +* Add proc-title-template option to control what's shown in the process title (#8397) +* Add lazyfree-lazy-user-flush config option to control FLUSHALL, FLUSHDB and SCRIPT FLUSH (#8258) + +Bug fixes: +* AOF: recover from last write error by turning on/off appendonly config (#8030) +* Exit on fsync error when the AOF fsync policy is 'always' (#8347) +* Avoid assertions (on older kernels) when testing arm64 CoW bug (#8405) +* CONFIG REWRITE should honor umask settings (#8371) +* Fix firstkey,lastkey,step in COMMAND command for some commands (#8367) + +Special considerations: +* Fix misleading description of the save configuration directive (#8337) + +Improvements: +* A way to get RDB file via replication without excessive replication buffers (#8303) +* Optimize performance of clusterGenNodesDescription for large clusters (#8182) + +Info fields and introspection changes: +* SLOWLOG and LATENCY monitor include unblocking time of blocked commands (#7491) + +Modules: +* Add modules API for streams (#8288) +* Add event for fork child birth and termination (#8289) +* Add RM_BlockedClientMeasureTime* etc, to track background processing in commandstats (#7491) +* Fix bug in v6.2, wrong value passed to the new unlink callback (#8381) +* Fix bug in v6.2, modules blocked on keys unblock on commands like LPUSH (#8356) + ================================================================================ Redis 6.2 RC2 Released Tue Jan 12 16:17:20 IST 2021 ================================================================================ @@ -255,35 +305,39 @@ and we don't get reports of serious issues for a while. A special thank you for the amount of work put into this release by: - Oran Agra - Yossi Gottlieb -- Itamar Haber -- Guy Benoish - Filipe Oliveira +- Viktor Söderqvist +- Guy Benoish +- Itamar Haber +- Yang Bodong - Madelyn Olson - Wang Yuan - Felipe Machado -- Yang Bodong +- Wen Hui - Tatsuya Arisawa - Jonah H. Harris +- Raghav Muddur - Jim Brunner - Yaacov Hazan -- Wen Hui +- Allen Farris - Chen Yang - Nitai Caro - Meir Shpilraien - maohuazhu - Valentino Geron -- Qu Chen -- sundb -- George Prekas - Zhao Zhao +- sundb +- Qu Chen +- George Prekas - Tyson Andre - Michael Grunder - alexronke-channeladvisor +- Andy Pan - Wu Yunlong - Wei Kukey - Yoav Steinberg -- Uri Shachar - Greg Femec +- Uri Shachar - Nykolas Laurentino de Lima - xhe - zhenwei pi diff --git a/redis.conf b/redis.conf index a5062fda9..465d56fc0 100644 --- a/redis.conf +++ b/redis.conf @@ -325,31 +325,52 @@ databases 16 # ASCII art logo in startup logs by setting the following option to yes. always-show-logo no -################################ SNAPSHOTTING ################################ -# -# Save the DB on disk: -# -# save -# -# Will save the DB if both the given number of seconds and the given -# number of write operations against the DB occurred. -# -# In the example below the behavior will be to save: -# after 900 sec (15 min) if at least 1 key changed -# after 300 sec (5 min) if at least 10 keys changed -# after 60 sec if at least 10000 keys changed -# -# Note: you can disable saving completely by commenting out all "save" lines. -# -# It is also possible to remove all the previously configured save -# points by adding a save directive with a single empty string argument -# like in the following example: -# -# save "" +# By default, Redis modifies the process title (as seen in 'top' and 'ps') to +# provide some runtime information. It is possible to disable this and leave +# the process name as executed by setting the following to no. +set-proc-title yes -save 900 1 -save 300 10 -save 60 10000 +# When changing the process title, Redis uses the following template to construct +# the modified title. +# +# Template variables are specified in curly brackets. The following variables are +# supported: +# +# {title} Name of process as executed if parent, or type of child process. +# {listen-addr} Bind address or '*' followed by TCP or TLS port listening on, or +# Unix socket if only that's available. +# {server-mode} Special mode, i.e. "[sentinel]" or "[cluster]". +# {port} TCP port listening on, or 0. +# {tls-port} TLS port listening on, or 0. +# {unixsocket} Unix domain socket listening on, or "". +# {config-file} Name of configuration file used. +# +proc-title-template "{title} {listen-addr} {server-mode}" + +################################ SNAPSHOTTING ################################ + +# Save the DB to disk. +# +# save +# +# Redis will save the DB if both the given number of seconds and the given +# number of write operations against the DB occurred. +# +# Snapshotting can be completely disabled with a single empty string argument +# as in following example: +# +# save "" +# +# Unless specified otherwise, by default Redis will save the DB: +# * After 3600 seconds (an hour) if at least 1 key changed +# * After 300 seconds (5 minutes) if at least 100 keys changed +# * After 60 seconds if at least 10000 keys changed +# +# You can set these explicitly by uncommenting the three following lines. +# +# save 3600 1 +# save 300 100 +# save 60 10000 # By default Redis will stop accepting writes if RDB snapshots are enabled # (at least one save point) and the latest background save failed. @@ -1089,6 +1110,13 @@ replica-lazy-flush no lazyfree-lazy-user-del no +# FLUSHDB, FLUSHALL, and SCRIPT FLUSH support both asynchronous and synchronous +# deletion, which can be controlled by passing the [SYNC|ASYNC] flags into the +# commands. When neither flag is passed, this directive will be used to determine +# if the data should be deleted asynchronously. + +lazyfree-lazy-user-flush no + ################################ THREADED I/O ################################# # Redis is mostly single threaded, however there are certain threaded diff --git a/runtest-moduleapi b/runtest-moduleapi index 9a48867d2..e554226c1 100755 --- a/runtest-moduleapi +++ b/runtest-moduleapi @@ -23,6 +23,7 @@ $TCLSH tests/test_helper.tcl \ --single unit/moduleapi/hooks \ --single unit/moduleapi/misc \ --single unit/moduleapi/blockonkeys \ +--single unit/moduleapi/blockonbackground \ --single unit/moduleapi/scan \ --single unit/moduleapi/datatype \ --single unit/moduleapi/auth \ @@ -31,4 +32,5 @@ $TCLSH tests/test_helper.tcl \ --single unit/moduleapi/getkeys \ --single unit/moduleapi/test_lazyfree \ --single unit/moduleapi/defrag \ +--single unit/moduleapi/stream \ "${@}" diff --git a/sentinel.conf b/sentinel.conf index 39d6929e7..8647379d8 100644 --- a/sentinel.conf +++ b/sentinel.conf @@ -321,3 +321,21 @@ sentinel deny-scripts-reconfig yes # is possible to just rename a command to itself: # # SENTINEL rename-command mymaster CONFIG CONFIG + +# HOSTNAMES SUPPORT +# +# Normally Sentinel uses only IP addresses and requires SENTINEL MONITOR +# to specify an IP address. Also, it requires the Redis replica-announce-ip +# keyword to specify only IP addresses. +# +# You may enable hostnames support by enabling resolve-hostnames. Note +# that you must make sure your DNS is configured properly and that DNS +# resolution does not introduce very long delays. +# +SENTINEL resolve-hostnames no + +# When resolve-hostnames is enabled, Sentinel still uses IP addresses +# when exposing instances to users, configuration files, etc. If you want +# to retain the hostnames when announced, enable announce-hostnames below. +# +SENTINEL announce-hostnames no diff --git a/src/acl.c b/src/acl.c index 14d023cc3..1f07c292f 100644 --- a/src/acl.c +++ b/src/acl.c @@ -1024,8 +1024,8 @@ int ACLSetUser(user *u, const char *op, ssize_t oplen) { /* Return a description of the error that occurred in ACLSetUser() according to * the errno value set by the function on error. */ -char *ACLSetUserStringError(void) { - char *errmsg = "Wrong format"; +const char *ACLSetUserStringError(void) { + const char *errmsg = "Wrong format"; if (errno == ENOENT) errmsg = "Unknown command or category name in ACL"; else if (errno == EINVAL) @@ -1454,7 +1454,7 @@ int ACLLoadConfiguredUsers(void) { /* Load every rule defined for this user. */ for (int j = 1; aclrules[j]; j++) { if (ACLSetUser(u,aclrules[j],sdslen(aclrules[j])) != C_OK) { - char *errmsg = ACLSetUserStringError(); + const char *errmsg = ACLSetUserStringError(); serverLog(LL_WARNING,"Error loading ACL rule '%s' for " "the user named '%s': %s", aclrules[j],aclrules[0],errmsg); @@ -1587,7 +1587,7 @@ sds ACLLoadFromFile(const char *filename) { for (j = 2; j < argc; j++) { argv[j] = sdstrim(argv[j],"\t\r\n"); if (ACLSetUser(fakeuser,argv[j],sdslen(argv[j])) != C_OK) { - char *errmsg = ACLSetUserStringError(); + const char *errmsg = ACLSetUserStringError(); errors = sdscatprintf(errors, "%s:%d: %s. ", server.acl_filename, linenum, errmsg); @@ -1908,7 +1908,7 @@ void aclCommand(client *c) { for (int j = 3; j < c->argc; j++) { if (ACLSetUser(tempu,c->argv[j]->ptr,sdslen(c->argv[j]->ptr)) != C_OK) { - char *errmsg = ACLSetUserStringError(); + const char *errmsg = ACLSetUserStringError(); addReplyErrorFormat(c, "Error in ACL SETUSER modifier '%s': %s", (char*)c->argv[j]->ptr, errmsg); diff --git a/src/ae.c b/src/ae.c index 1c3a4e091..283f51438 100644 --- a/src/ae.c +++ b/src/ae.c @@ -31,6 +31,7 @@ */ #include "ae.h" +#include "anet.h" #include #include diff --git a/src/ae_epoll.c b/src/ae_epoll.c index fa197297e..07ca8ca41 100644 --- a/src/ae_epoll.c +++ b/src/ae_epoll.c @@ -51,6 +51,7 @@ static int aeApiCreate(aeEventLoop *eventLoop) { zfree(state); return -1; } + anetCloexec(state->epfd); eventLoop->apidata = state; return 0; } diff --git a/src/ae_evport.c b/src/ae_evport.c index 4e254b602..7a0b03aea 100644 --- a/src/ae_evport.c +++ b/src/ae_evport.c @@ -82,6 +82,7 @@ static int aeApiCreate(aeEventLoop *eventLoop) { zfree(state); return -1; } + anetCloexec(state->portfd); state->npending = 0; diff --git a/src/ae_kqueue.c b/src/ae_kqueue.c index 6796f4ceb..b146f2519 100644 --- a/src/ae_kqueue.c +++ b/src/ae_kqueue.c @@ -53,6 +53,7 @@ static int aeApiCreate(aeEventLoop *eventLoop) { zfree(state); return -1; } + anetCloexec(state->kqfd); eventLoop->apidata = state; return 0; } diff --git a/src/anet.c b/src/anet.c index 7a0a1b1ed..0bfa575f5 100644 --- a/src/anet.c +++ b/src/anet.c @@ -69,6 +69,11 @@ int anetSetBlock(char *err, int fd, int non_block) { return ANET_ERR; } + /* Check if this flag has been set or unset, if so, + * then there is no need to call fcntl to set/unset it again. */ + if (!!(flags & O_NONBLOCK) == !!non_block) + return ANET_OK; + if (non_block) flags |= O_NONBLOCK; else @@ -89,6 +94,29 @@ int anetBlock(char *err, int fd) { return anetSetBlock(err,fd,0); } +/* Enable the FD_CLOEXEC on the given fd to avoid fd leaks. + * This function should be invoked for fd's on specific places + * where fork + execve system calls are called. */ +int anetCloexec(int fd) { + int r; + int flags; + + do { + r = fcntl(fd, F_GETFD); + } while (r == -1 && errno == EINTR); + + if (r == -1 || (r & FD_CLOEXEC)) + return r; + + flags = r | FD_CLOEXEC; + + do { + r = fcntl(fd, F_SETFD, flags); + } while (r == -1 && errno == EINTR); + + return r; +} + /* Set TCP keep alive option to detect dead peers. The interval option * is only used for Linux as we are using Linux-specific APIs to set * the probe send time, interval, and count. */ @@ -207,14 +235,13 @@ int anetRecvTimeout(char *err, int fd, long long ms) { return ANET_OK; } -/* anetGenericResolve() is called by anetResolve() and anetResolveIP() to - * do the actual work. It resolves the hostname "host" and set the string - * representation of the IP address into the buffer pointed by "ipbuf". +/* Resolve the hostname "host" and set the string representation of the + * IP address into the buffer pointed by "ipbuf". * * If flags is set to ANET_IP_ONLY the function only resolves hostnames * that are actually already IPv4 or IPv6 addresses. This turns the function * into a validating / normalizing function. */ -int anetGenericResolve(char *err, char *host, char *ipbuf, size_t ipbuf_len, +int anetResolve(char *err, char *host, char *ipbuf, size_t ipbuf_len, int flags) { struct addrinfo hints, *info; @@ -241,14 +268,6 @@ int anetGenericResolve(char *err, char *host, char *ipbuf, size_t ipbuf_len, return ANET_OK; } -int anetResolve(char *err, char *host, char *ipbuf, size_t ipbuf_len) { - return anetGenericResolve(err,host,ipbuf,ipbuf_len,ANET_NONE); -} - -int anetResolveIP(char *err, char *host, char *ipbuf, size_t ipbuf_len) { - return anetGenericResolve(err,host,ipbuf,ipbuf_len,ANET_IP_ONLY); -} - static int anetSetReuseAddr(char *err, int fd) { int yes = 1; /* Make sure connection-intensive things like the redis benchmark diff --git a/src/anet.h b/src/anet.h index fbf41cd17..5da2f3b46 100644 --- a/src/anet.h +++ b/src/anet.h @@ -60,8 +60,7 @@ int anetTcpNonBlockBestEffortBindConnect(char *err, const char *addr, int port, int anetUnixConnect(char *err, const char *path); int anetUnixNonBlockConnect(char *err, const char *path); int anetRead(int fd, char *buf, int count); -int anetResolve(char *err, char *host, char *ipbuf, size_t ipbuf_len); -int anetResolveIP(char *err, char *host, char *ipbuf, size_t ipbuf_len); +int anetResolve(char *err, char *host, char *ipbuf, size_t ipbuf_len, int flags); int anetTcpServer(char *err, int port, char *bindaddr, int backlog); int anetTcp6Server(char *err, int port, char *bindaddr, int backlog); int anetUnixServer(char *err, char *path, mode_t perm, int backlog); @@ -70,6 +69,7 @@ int anetUnixAccept(char *err, int serversock); int anetWrite(int fd, char *buf, int count); int anetNonBlock(char *err, int fd); int anetBlock(char *err, int fd); +int anetCloexec(int fd); int anetEnableTcpNoDelay(char *err, int fd); int anetDisableTcpNoDelay(char *err, int fd); int anetTcpKeepAlive(char *err, int fd); diff --git a/src/aof.c b/src/aof.c index d3191277f..6753e8bcc 100644 --- a/src/aof.c +++ b/src/aof.c @@ -235,6 +235,8 @@ void stopAppendOnly(void) { serverAssert(server.aof_state != AOF_OFF); flushAppendOnlyFile(1); redis_fsync(server.aof_fd); + server.aof_fsync_offset = server.aof_current_size; + server.aof_last_fsync = server.unixtime; close(server.aof_fd); server.aof_fd = -1; @@ -242,6 +244,8 @@ void stopAppendOnly(void) { server.aof_state = AOF_OFF; server.aof_rewrite_scheduled = 0; killAppendOnlyChild(); + sdsfree(server.aof_buf); + server.aof_buf = sdsempty(); } /* Called when the user switches from "appendonly no" to "appendonly yes" @@ -285,6 +289,12 @@ int startAppendOnly(void) { server.aof_state = AOF_WAIT_REWRITE; server.aof_last_fsync = server.unixtime; server.aof_fd = newfd; + + /* If AOF was in error state, we just ignore it and log the event. */ + if (server.aof_last_write_status == C_ERR) { + serverLog(LL_WARNING,"AOF reopen, just ignore the last error."); + server.aof_last_write_status = C_OK; + } return C_OK; } @@ -451,10 +461,11 @@ void flushAppendOnlyFile(int force) { /* Handle the AOF write error. */ if (server.aof_fsync == AOF_FSYNC_ALWAYS) { - /* We can't recover when the fsync policy is ALWAYS since the - * reply for the client is already in the output buffers, and we - * have the contract with the user that on acknowledged write data - * is synced on disk. */ + /* We can't recover when the fsync policy is ALWAYS since the reply + * for the client is already in the output buffers (both writes and + * reads), and the changes to the db can't be rolled back. Since we + * have a contract with the user that on acknowledged or observed + * writes are is synced on disk, we must exit. */ serverLog(LL_WARNING,"Can't recover from AOF write error when the AOF fsync policy is 'always'. Exiting..."); exit(1); } else { @@ -502,7 +513,14 @@ try_fsync: /* redis_fsync is defined as fdatasync() for Linux in order to avoid * flushing metadata. */ latencyStartMonitor(latency); - redis_fsync(server.aof_fd); /* Let's try to get this data on the disk */ + /* Let's try to get this data on the disk. To guarantee data safe when + * the AOF fsync policy is 'always', we should exit if failed to fsync + * AOF (see comment next to the exit(1) after write error above). */ + if (redis_fsync(server.aof_fd) == -1) { + serverLog(LL_WARNING,"Can't persist AOF for fsync error when the " + "AOF fsync policy is 'always': %s. Exiting...", strerror(errno)); + exit(1); + } latencyEndMonitor(latency); latencyAddSampleIfNeeded("aof-fsync-always",latency); server.aof_fsync_offset = server.aof_current_size; @@ -581,8 +599,6 @@ sds catAppendOnlyExpireAtCommand(sds buf, struct redisCommand *cmd, robj *key, r void feedAppendOnlyFile(struct redisCommand *cmd, int dictid, robj **argv, int argc) { sds buf = sdsempty(); - robj *tmpargv[3]; - /* The DB this command was targeting is not the same as the last command * we appended. To issue a SELECT command is needed. */ if (dictid != server.aof_selected_db) { @@ -598,32 +614,31 @@ void feedAppendOnlyFile(struct redisCommand *cmd, int dictid, robj **argv, int a cmd->proc == expireatCommand) { /* Translate EXPIRE/PEXPIRE/EXPIREAT into PEXPIREAT */ buf = catAppendOnlyExpireAtCommand(buf,cmd,argv[1],argv[2]); - } else if (cmd->proc == setexCommand || cmd->proc == psetexCommand) { - /* Translate SETEX/PSETEX to SET and PEXPIREAT */ - tmpargv[0] = createStringObject("SET",3); - tmpargv[1] = argv[1]; - tmpargv[2] = argv[3]; - buf = catAppendOnlyGenericCommand(buf,3,tmpargv); - decrRefCount(tmpargv[0]); - buf = catAppendOnlyExpireAtCommand(buf,cmd,argv[1],argv[2]); } else if (cmd->proc == setCommand && argc > 3) { - int i; - robj *exarg = NULL, *pxarg = NULL; - for (i = 3; i < argc; i ++) { - if (!strcasecmp(argv[i]->ptr, "ex")) exarg = argv[i+1]; - if (!strcasecmp(argv[i]->ptr, "px")) pxarg = argv[i+1]; + robj *pxarg = NULL; + /* When SET is used with EX/PX argument setGenericCommand propagates them with PX millisecond argument. + * So since the command arguments are re-written there, we can rely here on the index of PX being 3. */ + if (!strcasecmp(argv[3]->ptr, "px")) { + pxarg = argv[4]; } - serverAssert(!(exarg && pxarg)); + /* For AOF we convert SET key value relative time in milliseconds to SET key value absolute time in + * millisecond. Whenever the condition is true it implies that original SET has been transformed + * to SET PX with millisecond time argument so we do not need to worry about unit here.*/ + if (pxarg) { + robj *millisecond = getDecodedObject(pxarg); + long long when = strtoll(millisecond->ptr,NULL,10); + when += mstime(); - if (exarg || pxarg) { - /* Translate SET [EX seconds][PX milliseconds] to SET and PEXPIREAT */ - buf = catAppendOnlyGenericCommand(buf,3,argv); - if (exarg) - buf = catAppendOnlyExpireAtCommand(buf,server.expireCommand,argv[1], - exarg); - if (pxarg) - buf = catAppendOnlyExpireAtCommand(buf,server.pexpireCommand,argv[1], - pxarg); + decrRefCount(millisecond); + + robj *newargs[5]; + newargs[0] = argv[0]; + newargs[1] = argv[1]; + newargs[2] = argv[2]; + newargs[3] = shared.pxat; + newargs[4] = createStringObjectFromLongLong(when); + buf = catAppendOnlyGenericCommand(buf,5,newargs); + decrRefCount(newargs[4]); } else { buf = catAppendOnlyGenericCommand(buf,argc,argv); } @@ -1852,6 +1867,20 @@ void backgroundRewriteDoneHandler(int exitcode, int bysignal) { } latencyEndMonitor(latency); latencyAddSampleIfNeeded("aof-rewrite-diff-write",latency); + + if (server.aof_fsync == AOF_FSYNC_EVERYSEC) { + aof_background_fsync(newfd); + } else if (server.aof_fsync == AOF_FSYNC_ALWAYS) { + latencyStartMonitor(latency); + if (redis_fsync(newfd) == -1) { + serverLog(LL_WARNING, + "Error trying to fsync the parent diff to the rewritten AOF: %s", strerror(errno)); + close(newfd); + goto cleanup; + } + latencyEndMonitor(latency); + latencyAddSampleIfNeeded("aof-rewrite-done-fsync",latency); + } serverLog(LL_NOTICE, "Residual parent diff successfully flushed to the rewritten AOF (%.2f MB)", (double) aofRewriteBufferSize() / (1024*1024)); @@ -1919,14 +1948,11 @@ void backgroundRewriteDoneHandler(int exitcode, int bysignal) { /* AOF enabled, replace the old fd with the new one. */ oldfd = server.aof_fd; server.aof_fd = newfd; - if (server.aof_fsync == AOF_FSYNC_ALWAYS) - redis_fsync(newfd); - else if (server.aof_fsync == AOF_FSYNC_EVERYSEC) - aof_background_fsync(newfd); server.aof_selected_db = -1; /* Make sure SELECT is re-issued */ aofUpdateCurrentSize(); server.aof_rewrite_base_size = server.aof_current_size; server.aof_fsync_offset = server.aof_current_size; + server.aof_last_fsync = server.unixtime; /* Clear regular AOF buffer since its contents was just written to * the new AOF from the background rewrite buffer. */ diff --git a/src/blocked.c b/src/blocked.c index 46935c79f..09e17213c 100644 --- a/src/blocked.c +++ b/src/blocked.c @@ -61,6 +61,9 @@ */ #include "server.h" +#include "slowlog.h" +#include "latency.h" +#include "monotonic.h" int serveClientBlockedOnList(client *receiver, robj *key, robj *dstkey, redisDb *db, robj *value, int wherefrom, int whereto); int getListPositionFromObjectOrReply(client *c, robj *arg, int *position); @@ -97,6 +100,20 @@ void blockClient(client *c, int btype) { } } +/* This function is called after a client has finished a blocking operation + * in order to update the total command duration, log the command into + * the Slow log if needed, and log the reply duration event if needed. */ +void updateStatsOnUnblock(client *c, long blocked_us, long reply_us){ + const ustime_t total_cmd_duration = c->duration + blocked_us + reply_us; + c->lastcmd->microseconds += total_cmd_duration; + /* Log the command into the Slow log if needed. */ + if (!(c->lastcmd->flags & CMD_SKIP_SLOWLOG)) { + slowlogPushEntryIfNeeded(c,c->argv,c->argc,total_cmd_duration); + /* Log the reply duration event. */ + latencyAddSampleIfNeeded("command-unblocking",reply_us/1000); + } +} + /* This function is called in the beforeSleep() function of the event loop * in order to process the pending input buffer of clients that were * unblocked after a blocking operation. */ @@ -264,6 +281,8 @@ void serveClientsBlockedOnListKey(robj *o, readyList *rl) { if (dstkey) incrRefCount(dstkey); unblockClient(receiver); + monotime replyTimer; + elapsedStart(&replyTimer); if (serveClientBlockedOnList(receiver, rl->key,dstkey,rl->db,value, wherefrom, whereto) == C_ERR) @@ -272,6 +291,7 @@ void serveClientsBlockedOnListKey(robj *o, readyList *rl) { * to also undo the POP operation. */ listTypePush(o,value,wherefrom); } + updateStatsOnUnblock(receiver, 0, elapsedUs(replyTimer)); if (dstkey) decrRefCount(dstkey); decrRefCount(value); @@ -316,7 +336,10 @@ void serveClientsBlockedOnSortedSetKey(robj *o, readyList *rl) { receiver->lastcmd->proc == bzpopminCommand) ? ZSET_MIN : ZSET_MAX; unblockClient(receiver); + monotime replyTimer; + elapsedStart(&replyTimer); genericZpopCommand(receiver,&rl->key,1,where,1,NULL); + updateStatsOnUnblock(receiver, 0, elapsedUs(replyTimer)); zcard--; /* Replicate the command. */ @@ -406,6 +429,8 @@ void serveClientsBlockedOnStreamKey(robj *o, readyList *rl) { } } + monotime replyTimer; + elapsedStart(&replyTimer); /* Emit the two elements sub-array consisting of * the name of the stream and the data we * extracted from it. Wrapped in a single-item @@ -425,6 +450,7 @@ void serveClientsBlockedOnStreamKey(robj *o, readyList *rl) { streamReplyWithRange(receiver,s,&start,NULL, receiver->bpop.xread_count, 0, group, consumer, noack, &pi); + updateStatsOnUnblock(receiver, 0, elapsedUs(replyTimer)); /* Note that after we unblock the client, 'gt' * and other receiver->bpop stuff are no longer @@ -471,7 +497,10 @@ void serveClientsBlockedOnKeyByModule(readyList *rl) { * different modules with different triggers to consider if a key * is ready or not. This means we can't exit the loop but need * to continue after the first failure. */ + monotime replyTimer; + elapsedStart(&replyTimer); if (!moduleTryServeClientBlockedOnKey(receiver, rl->key)) continue; + updateStatsOnUnblock(receiver, 0, elapsedUs(replyTimer)); moduleUnblockClient(receiver); } @@ -684,10 +713,20 @@ static int getBlockedTypeByType(int type) { void signalKeyAsReady(redisDb *db, robj *key, int type) { readyList *rl; - /* If no clients are blocked on this type, just return */ + /* Quick returns. */ int btype = getBlockedTypeByType(type); - if (btype == BLOCKED_NONE || !server.blocked_clients_by_type[btype]) + if (btype == BLOCKED_NONE) { + /* The type can never block. */ return; + } + if (!server.blocked_clients_by_type[btype] && + !server.blocked_clients_by_type[BLOCKED_MODULE]) { + /* No clients block on this type. Note: Blocked modules are represented + * by BLOCKED_MODULE, even if the intention is to wake up by normal + * types (list, zset, stream), so we need to check that there are no + * blocked modules before we do a quick return here. */ + return; + } /* No clients blocking for this key? No need to queue it. */ if (dictFind(db->blocking_keys,key) == NULL) return; diff --git a/src/cluster.c b/src/cluster.c index 78c36e8d1..97a25b0b3 100644 --- a/src/cluster.c +++ b/src/cluster.c @@ -398,7 +398,7 @@ int clusterLockConfig(char *filename) { /* To lock it, we need to open the file in a way it is created if * it does not exist, otherwise there is a race condition with other * processes. */ - int fd = open(filename,O_WRONLY|O_CREAT,0644); + int fd = open(filename,O_WRONLY|O_CREAT|O_CLOEXEC,0644); if (fd == -1) { serverLog(LL_WARNING, "Can't open %s in order to acquire a lock: %s", @@ -509,8 +509,7 @@ void clusterInit(void) { serverLog(LL_WARNING, "Redis port number too high. " "Cluster communication port is 10,000 port " "numbers higher than your Redis port. " - "Your Redis port number must be " - "lower than 55535."); + "Your Redis port number must be 55535 or less."); exit(1); } if (listenToPort(port+CLUSTER_PORT_INCR, @@ -779,6 +778,7 @@ clusterNode *createClusterNode(char *nodename, int flags) { node->configEpoch = 0; node->flags = flags; memset(node->slots,0,sizeof(node->slots)); + node->slots_info = NULL; node->numslots = 0; node->numslaves = 0; node->slaves = NULL; @@ -4144,8 +4144,8 @@ sds clusterGenNodeDescription(clusterNode *node) { sds ci; /* Node coordinates */ - ci = sdscatprintf(sdsempty(),"%.40s %s:%d@%d ", - node->name, + ci = sdscatlen(sdsempty(),node->name,CLUSTER_NAMELEN); + ci = sdscatfmt(ci," %s:%i@%i ", node->ip, node->port, node->cport); @@ -4154,40 +4154,46 @@ sds clusterGenNodeDescription(clusterNode *node) { ci = representClusterNodeFlags(ci, node->flags); /* Slave of... or just "-" */ + ci = sdscatlen(ci," ",1); if (node->slaveof) - ci = sdscatprintf(ci," %.40s ",node->slaveof->name); + ci = sdscatlen(ci,node->slaveof->name,CLUSTER_NAMELEN); else - ci = sdscatlen(ci," - ",3); + ci = sdscatlen(ci,"-",1); unsigned long long nodeEpoch = node->configEpoch; if (nodeIsSlave(node) && node->slaveof) { nodeEpoch = node->slaveof->configEpoch; } /* Latency from the POV of this node, config epoch, link status */ - ci = sdscatprintf(ci,"%lld %lld %llu %s", + ci = sdscatfmt(ci," %I %I %U %s", (long long) node->ping_sent, (long long) node->pong_received, nodeEpoch, (node->link || node->flags & CLUSTER_NODE_MYSELF) ? "connected" : "disconnected"); - /* Slots served by this instance */ - start = -1; - for (j = 0; j < CLUSTER_SLOTS; j++) { - int bit; + /* Slots served by this instance. If we already have slots info, + * append it diretly, otherwise, generate slots only if it has. */ + if (node->slots_info) { + ci = sdscatsds(ci, node->slots_info); + } else if (node->numslots > 0) { + start = -1; + for (j = 0; j < CLUSTER_SLOTS; j++) { + int bit; - if ((bit = clusterNodeGetSlotBit(node,j)) != 0) { - if (start == -1) start = j; - } - if (start != -1 && (!bit || j == CLUSTER_SLOTS-1)) { - if (bit && j == CLUSTER_SLOTS-1) j++; - - if (start == j-1) { - ci = sdscatprintf(ci," %d",start); - } else { - ci = sdscatprintf(ci," %d-%d",start,j-1); + if ((bit = clusterNodeGetSlotBit(node,j)) != 0) { + if (start == -1) start = j; + } + if (start != -1 && (!bit || j == CLUSTER_SLOTS-1)) { + if (bit && j == CLUSTER_SLOTS-1) j++; + + if (start == j-1) { + ci = sdscatfmt(ci," %i",start); + } else { + ci = sdscatfmt(ci," %i-%i",start,j-1); + } + start = -1; } - start = -1; } } @@ -4208,6 +4214,41 @@ sds clusterGenNodeDescription(clusterNode *node) { return ci; } +/* Generate the slot topology for all nodes and store the string representation + * in the slots_info struct on the node. This is used to improve the efficiency + * of clusterGenNodesDescription() because it removes looping of the slot space + * for generating the slot info for each node individually. */ +void clusterGenNodesSlotsInfo(int filter) { + clusterNode *n = NULL; + int start = -1; + + for (int i = 0; i <= CLUSTER_SLOTS; i++) { + /* Find start node and slot id. */ + if (n == NULL) { + if (i == CLUSTER_SLOTS) break; + n = server.cluster->slots[i]; + start = i; + continue; + } + + /* Generate slots info when occur different node with start + * or end of slot. */ + if (i == CLUSTER_SLOTS || n != server.cluster->slots[i]) { + if (!(n->flags & filter)) { + if (n->slots_info == NULL) n->slots_info = sdsempty(); + if (start == i-1) { + n->slots_info = sdscatfmt(n->slots_info," %i",start); + } else { + n->slots_info = sdscatfmt(n->slots_info," %i-%i",start,i-1); + } + } + if (i == CLUSTER_SLOTS) break; + n = server.cluster->slots[i]; + start = i; + } + } +} + /* Generate a csv-alike representation of the nodes we are aware of, * including the "myself" node, and return an SDS string containing the * representation (it is up to the caller to free it). @@ -4225,6 +4266,9 @@ sds clusterGenNodesDescription(int filter) { dictIterator *di; dictEntry *de; + /* Generate all nodes slots info firstly. */ + clusterGenNodesSlotsInfo(filter); + di = dictGetSafeIterator(server.cluster->nodes); while((de = dictNext(di)) != NULL) { clusterNode *node = dictGetVal(de); @@ -4234,6 +4278,12 @@ sds clusterGenNodesDescription(int filter) { ci = sdscatsds(ci,ni); sdsfree(ni); ci = sdscatlen(ci,"\n",1); + + /* Release slots info. */ + if (node->slots_info) { + sdsfree(node->slots_info); + node->slots_info = NULL; + } } dictReleaseIterator(di); return ci; diff --git a/src/cluster.h b/src/cluster.h index d58f350ce..716c0d49c 100644 --- a/src/cluster.h +++ b/src/cluster.h @@ -118,6 +118,7 @@ typedef struct clusterNode { int flags; /* CLUSTER_NODE_... */ uint64_t configEpoch; /* Last configEpoch observed for this node */ unsigned char slots[CLUSTER_SLOTS/8]; /* slots handled by this node */ + sds slots_info; /* Slots info represented by string. */ int numslots; /* Number of slots handled by this node */ int numslaves; /* Number of slave nodes, if this is a master */ struct clusterNode **slaves; /* pointers to slave nodes */ diff --git a/src/config.c b/src/config.c index 2e109dbae..0bd89c2b9 100644 --- a/src/config.c +++ b/src/config.c @@ -153,15 +153,15 @@ int configOOMScoreAdjValuesDefaults[CONFIG_OOM_COUNT] = { 0, 200, 800 }; typedef struct boolConfigData { int *config; /* The pointer to the server config this value is stored in */ const int default_value; /* The default value of the config on rewrite */ - int (*is_valid_fn)(int val, char **err); /* Optional function to check validity of new value (generic doc above) */ - int (*update_fn)(int val, int prev, char **err); /* Optional function to apply new value at runtime (generic doc above) */ + int (*is_valid_fn)(int val, const char **err); /* Optional function to check validity of new value (generic doc above) */ + int (*update_fn)(int val, int prev, const char **err); /* Optional function to apply new value at runtime (generic doc above) */ } boolConfigData; typedef struct stringConfigData { char **config; /* Pointer to the server config this value is stored in. */ const char *default_value; /* Default value of the config on rewrite. */ - int (*is_valid_fn)(char* val, char **err); /* Optional function to check validity of new value (generic doc above) */ - int (*update_fn)(char* val, char* prev, char **err); /* Optional function to apply new value at runtime (generic doc above) */ + int (*is_valid_fn)(char* val, const char **err); /* Optional function to check validity of new value (generic doc above) */ + int (*update_fn)(char* val, char* prev, const char **err); /* Optional function to apply new value at runtime (generic doc above) */ int convert_empty_to_null; /* Boolean indicating if empty strings should be stored as a NULL value. */ } stringConfigData; @@ -169,8 +169,8 @@ typedef struct stringConfigData { typedef struct sdsConfigData { sds *config; /* Pointer to the server config this value is stored in. */ const char *default_value; /* Default value of the config on rewrite. */ - int (*is_valid_fn)(sds val, char **err); /* Optional function to check validity of new value (generic doc above) */ - int (*update_fn)(sds val, sds prev, char **err); /* Optional function to apply new value at runtime (generic doc above) */ + int (*is_valid_fn)(sds val, const char **err); /* Optional function to check validity of new value (generic doc above) */ + int (*update_fn)(sds val, sds prev, const char **err); /* Optional function to apply new value at runtime (generic doc above) */ int convert_empty_to_null; /* Boolean indicating if empty SDS strings should be stored as a NULL value. */ } sdsConfigData; @@ -179,8 +179,8 @@ typedef struct enumConfigData { int *config; /* The pointer to the server config this value is stored in */ configEnum *enum_value; /* The underlying enum type this data represents */ const int default_value; /* The default value of the config on rewrite */ - int (*is_valid_fn)(int val, char **err); /* Optional function to check validity of new value (generic doc above) */ - int (*update_fn)(int val, int prev, char **err); /* Optional function to apply new value at runtime (generic doc above) */ + int (*is_valid_fn)(int val, const char **err); /* Optional function to check validity of new value (generic doc above) */ + int (*update_fn)(int val, int prev, const char **err); /* Optional function to apply new value at runtime (generic doc above) */ } enumConfigData; typedef enum numericType { @@ -214,8 +214,8 @@ typedef struct numericConfigData { long long lower_bound; /* The lower bound of this numeric value */ long long upper_bound; /* The upper bound of this numeric value */ const long long default_value; /* The default value of the config on rewrite */ - int (*is_valid_fn)(long long val, char **err); /* Optional function to check validity of new value (generic doc above) */ - int (*update_fn)(long long val, long long prev, char **err); /* Optional function to apply new value at runtime (generic doc above) */ + int (*is_valid_fn)(long long val, const char **err); /* Optional function to check validity of new value (generic doc above) */ + int (*update_fn)(long long val, long long prev, const char **err); /* Optional function to apply new value at runtime (generic doc above) */ } numericConfigData; typedef union typeData { @@ -230,10 +230,10 @@ typedef struct typeInterface { /* Called on server start, to init the server with default value */ void (*init)(typeData data); /* Called on server start, should return 1 on success, 0 on error and should set err */ - int (*load)(typeData data, sds *argc, int argv, char **err); + int (*load)(typeData data, sds *argc, int argv, const char **err); /* Called on server startup and CONFIG SET, returns 1 on success, 0 on error * and can set a verbose err string, update is true when called from CONFIG SET */ - int (*set)(typeData data, sds value, int update, char **err); + int (*set)(typeData data, sds value, int update, const char **err); /* Called on CONFIG GET, required to add output to the client */ void (*get)(client *c, typeData data); /* Called on CONFIG REWRITE, required to rewrite the config state */ @@ -325,7 +325,7 @@ void queueLoadModule(sds path, sds *argv, int argc) { * server.oom_score_adj_values if valid. */ -static int updateOOMScoreAdjValues(sds *args, char **err, int apply) { +static int updateOOMScoreAdjValues(sds *args, const char **err, int apply) { int i; int values[CONFIG_OOM_COUNT]; @@ -385,7 +385,7 @@ void initConfigValues() { } void loadServerConfigFromString(char *config) { - char *err = NULL; + const char *err = NULL; int linenum = 0, totlines, i; int slaveof_linenum = 0; sds *lines; @@ -608,7 +608,7 @@ void loadServerConfigFromString(char *config) { int argc_err; if (ACLAppendUserForLoading(argv,argc,&argc_err) == C_ERR) { char buf[1024]; - char *errmsg = ACLSetUserStringError(); + const char *errmsg = ACLSetUserStringError(); snprintf(buf,sizeof(buf),"Error in user declaration '%s': %s", argv[argc_err],errmsg); err = buf; @@ -624,8 +624,7 @@ void loadServerConfigFromString(char *config) { err = "sentinel directive while not in sentinel mode"; goto loaderr; } - err = sentinelHandleConfiguration(argv+1,argc-1); - if (err) goto loaderr; + queueSentinelConfig(argv+1,argc-1,linenum,lines[i]); } } else { err = "Bad directive or wrong number of arguments"; goto loaderr; @@ -730,7 +729,7 @@ void configSetCommand(client *c) { robj *o; long long ll; int err; - char *errstr = NULL; + const char *errstr = NULL; serverAssertWithInfo(c,c->argv[2],sdsEncodedObject(c->argv[2])); serverAssertWithInfo(c,c->argv[3],sdsEncodedObject(c->argv[3])); o = c->argv[3]; @@ -1221,7 +1220,16 @@ struct rewriteConfigState *rewriteConfigReadOldFile(char *path) { sdsfree(argv[0]); argv[0] = alt; } - rewriteConfigAddLineNumberToOption(state,argv[0],linenum); + /* If this is sentinel config, we use sentinel "sentinel " as option + to avoid messing up the sequence. */ + if (server.sentinel_mode && argc > 1 && !strcasecmp(argv[0],"sentinel")) { + sds sentinelOption = sdsempty(); + sentinelOption = sdscatfmt(sentinelOption,"%S %S",argv[0],argv[1]); + rewriteConfigAddLineNumberToOption(state,sentinelOption,linenum); + sdsfree(sentinelOption); + } else { + rewriteConfigAddLineNumberToOption(state,argv[0],linenum); + } sdsfreesplitres(argv,argc); } fclose(fp); @@ -1683,7 +1691,7 @@ int rewriteConfigOverwriteFile(char *configfile, sds content) { if (fsync(fd)) serverLog(LL_WARNING, "Could not sync tmp config file to disk (%s)", strerror(errno)); - else if (fchmod(fd, 0644) == -1) + else if (fchmod(fd, 0644 & ~server.umask) == -1) serverLog(LL_WARNING, "Could not chmod config file (%s)", strerror(errno)); else if (rename(tmp_conffile, configfile) == -1) serverLog(LL_WARNING, "Could not rename tmp config file (%s)", strerror(errno)); @@ -1795,7 +1803,7 @@ static void boolConfigInit(typeData data) { *data.yesno.config = data.yesno.default_value; } -static int boolConfigSet(typeData data, sds value, int update, char **err) { +static int boolConfigSet(typeData data, sds value, int update, const char **err) { int yn = yesnotoi(value); if (yn == -1) { *err = "argument must be 'yes' or 'no'"; @@ -1836,7 +1844,7 @@ static void stringConfigInit(typeData data) { *data.string.config = (data.string.convert_empty_to_null && !data.string.default_value) ? NULL : zstrdup(data.string.default_value); } -static int stringConfigSet(typeData data, sds value, int update, char **err) { +static int stringConfigSet(typeData data, sds value, int update, const char **err) { if (data.string.is_valid_fn && !data.string.is_valid_fn(value, err)) return 0; char *prev = *data.string.config; @@ -1863,7 +1871,7 @@ static void sdsConfigInit(typeData data) { *data.sds.config = (data.sds.convert_empty_to_null && !data.sds.default_value) ? NULL: sdsnew(data.sds.default_value); } -static int sdsConfigSet(typeData data, sds value, int update, char **err) { +static int sdsConfigSet(typeData data, sds value, int update, const char **err) { if (data.sds.is_valid_fn && !data.sds.is_valid_fn(value, err)) return 0; sds prev = *data.sds.config; @@ -1922,7 +1930,7 @@ static void enumConfigInit(typeData data) { *data.enumd.config = data.enumd.default_value; } -static int enumConfigSet(typeData data, sds value, int update, char **err) { +static int enumConfigSet(typeData data, sds value, int update, const char **err) { int enumval = configEnumGetValue(data.enumd.enum_value, value); if (enumval == INT_MIN) { sds enumerr = sdsnew("argument must be one of the following: "); @@ -2028,7 +2036,7 @@ static void numericConfigInit(typeData data) { SET_NUMERIC_TYPE(data.numeric.default_value) } -static int numericBoundaryCheck(typeData data, long long ll, char **err) { +static int numericBoundaryCheck(typeData data, long long ll, const char **err) { if (data.numeric.numeric_type == NUMERIC_TYPE_ULONG_LONG || data.numeric.numeric_type == NUMERIC_TYPE_UINT || data.numeric.numeric_type == NUMERIC_TYPE_SIZE_T) { @@ -2058,7 +2066,7 @@ static int numericBoundaryCheck(typeData data, long long ll, char **err) { return 1; } -static int numericConfigSet(typeData data, sds value, int update, char **err) { +static int numericConfigSet(typeData data, sds value, int update, const char **err) { long long ll, prev = 0; if (data.numeric.is_memory) { int memerr; @@ -2196,7 +2204,7 @@ static void numericConfigRewrite(typeData data, const char *name, struct rewrite } \ } -static int isValidActiveDefrag(int val, char **err) { +static int isValidActiveDefrag(int val, const char **err) { #ifndef HAVE_DEFRAG if (val) { *err = "Active defragmentation cannot be enabled: it " @@ -2212,7 +2220,7 @@ static int isValidActiveDefrag(int val, char **err) { return 1; } -static int isValidDBfilename(char *val, char **err) { +static int isValidDBfilename(char *val, const char **err) { if (!pathIsBaseName(val)) { *err = "dbfilename can't be a path, just a filename"; return 0; @@ -2220,7 +2228,7 @@ static int isValidDBfilename(char *val, char **err) { return 1; } -static int isValidAOFfilename(char *val, char **err) { +static int isValidAOFfilename(char *val, const char **err) { if (!pathIsBaseName(val)) { *err = "appendfilename can't be a path, just a filename"; return 0; @@ -2228,7 +2236,26 @@ static int isValidAOFfilename(char *val, char **err) { return 1; } -static int updateHZ(long long val, long long prev, char **err) { +/* Validate specified string is a valid proc-title-template */ +static int isValidProcTitleTemplate(char *val, const char **err) { + if (!validateProcTitleTemplate(val)) { + *err = "template format is invalid or contains unknown variables"; + return 0; + } + return 1; +} + +static int updateProcTitleTemplate(char *val, char *prev, const char **err) { + UNUSED(val); + UNUSED(prev); + if (redisSetProcTitle(NULL) == C_ERR) { + *err = "failed to set process title"; + return 0; + } + return 1; +} + +static int updateHZ(long long val, long long prev, const char **err) { UNUSED(prev); UNUSED(err); /* Hz is more a hint from the user, so we accept values out of range @@ -2240,14 +2267,14 @@ static int updateHZ(long long val, long long prev, char **err) { return 1; } -static int updateJemallocBgThread(int val, int prev, char **err) { +static int updateJemallocBgThread(int val, int prev, const char **err) { UNUSED(prev); UNUSED(err); set_jemalloc_bg_thread(val); return 1; } -static int updateReplBacklogSize(long long val, long long prev, char **err) { +static int updateReplBacklogSize(long long val, long long prev, const char **err) { /* resizeReplicationBacklog sets server.repl_backlog_size, and relies on * being able to tell when the size changes, so restore prev before calling it. */ UNUSED(err); @@ -2256,7 +2283,7 @@ static int updateReplBacklogSize(long long val, long long prev, char **err) { return 1; } -static int updateMaxmemory(long long val, long long prev, char **err) { +static int updateMaxmemory(long long val, long long prev, const char **err) { UNUSED(prev); UNUSED(err); if (val) { @@ -2269,7 +2296,7 @@ static int updateMaxmemory(long long val, long long prev, char **err) { return 1; } -static int updateGoodSlaves(long long val, long long prev, char **err) { +static int updateGoodSlaves(long long val, long long prev, const char **err) { UNUSED(val); UNUSED(prev); UNUSED(err); @@ -2277,7 +2304,7 @@ static int updateGoodSlaves(long long val, long long prev, char **err) { return 1; } -static int updateAppendonly(int val, int prev, char **err) { +static int updateAppendonly(int val, int prev, const char **err) { UNUSED(prev); if (val == 0 && server.aof_state != AOF_OFF) { stopAppendOnly(); @@ -2290,7 +2317,7 @@ static int updateAppendonly(int val, int prev, char **err) { return 1; } -static int updateSighandlerEnabled(int val, int prev, char **err) { +static int updateSighandlerEnabled(int val, int prev, const char **err) { UNUSED(err); UNUSED(prev); if (val) @@ -2300,7 +2327,7 @@ static int updateSighandlerEnabled(int val, int prev, char **err) { return 1; } -static int updateMaxclients(long long val, long long prev, char **err) { +static int updateMaxclients(long long val, long long prev, const char **err) { /* Try to check if the OS is capable of supporting so many FDs. */ if (val > prev) { adjustOpenFilesLimit(); @@ -2328,7 +2355,7 @@ static int updateMaxclients(long long val, long long prev, char **err) { return 1; } -static int updateOOMScoreAdj(int val, int prev, char **err) { +static int updateOOMScoreAdj(int val, int prev, const char **err) { UNUSED(prev); if (val) { @@ -2342,7 +2369,7 @@ static int updateOOMScoreAdj(int val, int prev, char **err) { } #ifdef USE_OPENSSL -static int updateTlsCfg(char *val, char *prev, char **err) { +static int updateTlsCfg(char *val, char *prev, const char **err) { UNUSED(val); UNUSED(prev); UNUSED(err); @@ -2355,13 +2382,13 @@ static int updateTlsCfg(char *val, char *prev, char **err) { } return 1; } -static int updateTlsCfgBool(int val, int prev, char **err) { +static int updateTlsCfgBool(int val, int prev, const char **err) { UNUSED(val); UNUSED(prev); return updateTlsCfg(NULL, NULL, err); } -static int updateTlsCfgInt(long long val, long long prev, char **err) { +static int updateTlsCfgInt(long long val, long long prev, const char **err) { UNUSED(val); UNUSED(prev); return updateTlsCfg(NULL, NULL, err); @@ -2380,11 +2407,13 @@ standardConfig configs[] = { createBoolConfig("rdb-del-sync-files", NULL, MODIFIABLE_CONFIG, server.rdb_del_sync_files, 0, NULL, NULL), createBoolConfig("activerehashing", NULL, MODIFIABLE_CONFIG, server.activerehashing, 1, NULL, NULL), createBoolConfig("stop-writes-on-bgsave-error", NULL, MODIFIABLE_CONFIG, server.stop_writes_on_bgsave_err, 1, NULL, NULL), + createBoolConfig("set-proc-title", NULL, IMMUTABLE_CONFIG, server.set_proc_title, 1, NULL, NULL), /* Should setproctitle be used? */ createBoolConfig("dynamic-hz", NULL, MODIFIABLE_CONFIG, server.dynamic_hz, 1, NULL, NULL), /* Adapt hz to # of clients.*/ createBoolConfig("lazyfree-lazy-eviction", NULL, MODIFIABLE_CONFIG, server.lazyfree_lazy_eviction, 0, NULL, NULL), createBoolConfig("lazyfree-lazy-expire", NULL, MODIFIABLE_CONFIG, server.lazyfree_lazy_expire, 0, NULL, NULL), createBoolConfig("lazyfree-lazy-server-del", NULL, MODIFIABLE_CONFIG, server.lazyfree_lazy_server_del, 0, NULL, NULL), createBoolConfig("lazyfree-lazy-user-del", NULL, MODIFIABLE_CONFIG, server.lazyfree_lazy_user_del , 0, NULL, NULL), + createBoolConfig("lazyfree-lazy-user-flush", NULL, MODIFIABLE_CONFIG, server.lazyfree_lazy_user_flush , 0, NULL, NULL), createBoolConfig("repl-disable-tcp-nodelay", NULL, MODIFIABLE_CONFIG, server.repl_disable_tcp_nodelay, 0, NULL, NULL), createBoolConfig("repl-diskless-sync", NULL, MODIFIABLE_CONFIG, server.repl_diskless_sync, 0, NULL, NULL), createBoolConfig("gopher-enabled", NULL, MODIFIABLE_CONFIG, server.gopher_enabled, 0, NULL, NULL), @@ -2425,6 +2454,7 @@ standardConfig configs[] = { createStringConfig("aof_rewrite_cpulist", NULL, IMMUTABLE_CONFIG, EMPTY_STRING_IS_NULL, server.aof_rewrite_cpulist, NULL, NULL, NULL), createStringConfig("bgsave_cpulist", NULL, IMMUTABLE_CONFIG, EMPTY_STRING_IS_NULL, server.bgsave_cpulist, NULL, NULL, NULL), createStringConfig("ignore-warnings", NULL, MODIFIABLE_CONFIG, ALLOW_EMPTY_STRING, server.ignore_warnings, "", NULL, NULL), + createStringConfig("proc-title-template", NULL, MODIFIABLE_CONFIG, ALLOW_EMPTY_STRING, server.proc_title_template, CONFIG_DEFAULT_PROC_TITLE_TEMPLATE, isValidProcTitleTemplate, updateProcTitleTemplate), /* SDS Configs */ createSDSConfig("masterauth", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.masterauth, NULL, NULL, NULL), diff --git a/src/db.c b/src/db.c index 5d63566a7..3871753dd 100644 --- a/src/db.c +++ b/src/db.c @@ -226,7 +226,7 @@ void dbOverwrite(redisDb *db, robj *key, robj *val) { /* Although the key is not really deleted from the database, we regard overwrite as two steps of unlink+add, so we still need to call the unlink callback of the module. */ - moduleNotifyKeyUnlink(key,val); + moduleNotifyKeyUnlink(key,old); dictSetVal(db->dict, de, val); if (server.lazyfree_lazy_server_del) { @@ -595,21 +595,23 @@ void signalFlushedDb(int dbid, int async) { /* Return the set of flags to use for the emptyDb() call for FLUSHALL * and FLUSHDB commands. * - * Currently the command just attempts to parse the "ASYNC" option. It - * also checks if the command arity is wrong. + * sync: flushes the database in an sync manner. + * async: flushes the database in an async manner. + * no option: determine sync or async according to the value of lazyfree-lazy-user-flush. * * On success C_OK is returned and the flags are stored in *flags, otherwise * C_ERR is returned and the function sends an error to the client. */ int getFlushCommandFlags(client *c, int *flags) { /* Parse the optional ASYNC option. */ - if (c->argc > 1) { - if (c->argc > 2 || strcasecmp(c->argv[1]->ptr,"async")) { - addReplyErrorObject(c,shared.syntaxerr); - return C_ERR; - } - *flags = EMPTYDB_ASYNC; - } else { + if (c->argc == 2 && !strcasecmp(c->argv[1]->ptr,"sync")) { *flags = EMPTYDB_NO_FLAGS; + } else if (c->argc == 2 && !strcasecmp(c->argv[1]->ptr,"async")) { + *flags = EMPTYDB_ASYNC; + } else if (c->argc == 1) { + *flags = server.lazyfree_lazy_user_flush ? EMPTYDB_ASYNC : EMPTYDB_NO_FLAGS; + } else { + addReplyErrorObject(c,shared.syntaxerr); + return C_ERR; } return C_OK; } @@ -951,7 +953,7 @@ void scanGenericCommand(client *c, robj *o, unsigned long cursor) { int filter = 0; /* Filter element if it does not match the pattern. */ - if (!filter && use_pattern) { + if (use_pattern) { if (sdsEncodedObject(kobj)) { if (!stringmatchlen(pat, patlen, kobj->ptr, sdslen(kobj->ptr), 0)) filter = 1; diff --git a/src/defrag.c b/src/defrag.c index e189deddd..db797711e 100644 --- a/src/defrag.c +++ b/src/defrag.c @@ -367,7 +367,7 @@ long activeDefragSdsListAndDict(list *l, dict *d, int dict_val_type) { } else if (dict_val_type == DEFRAG_SDS_DICT_VAL_VOID_PTR) { void *newptr, *ptr = dictGetVal(de); if ((newptr = activeDefragAlloc(ptr))) - ln->value = newptr, defragged++; + de->v.val = newptr, defragged++; } defragged += dictIterDefragEntry(di); } diff --git a/src/expire.c b/src/expire.c index 275a735a7..f79510817 100644 --- a/src/expire.c +++ b/src/expire.c @@ -53,15 +53,19 @@ * to the function to avoid too many gettimeofday() syscalls. */ int activeExpireCycleTryExpire(redisDb *db, dictEntry *de, long long now) { long long t = dictGetSignedIntegerVal(de); + mstime_t expire_latency; if (now > t) { sds key = dictGetKey(de); robj *keyobj = createStringObject(key,sdslen(key)); propagateExpire(db,keyobj,server.lazyfree_lazy_expire); + latencyStartMonitor(expire_latency); if (server.lazyfree_lazy_expire) dbAsyncDelete(db,keyobj); else dbSyncDelete(db,keyobj); + latencyEndMonitor(expire_latency); + latencyAddSampleIfNeeded("expire-del",expire_latency); notifyKeyspaceEvent(NOTIFY_EXPIRED, "expired",keyobj,db->id); signalModifiedKey(NULL, db, keyobj); @@ -224,7 +228,7 @@ void activeExpireCycle(int type) { /* When there are less than 1% filled slots, sampling the key * space is expensive, so stop here waiting for better times... * The dictionary will be resized asap. */ - if (num && slots > DICT_HT_INITIAL_SIZE && + if (slots > DICT_HT_INITIAL_SIZE && (num*100/slots < 1)) break; /* The main collection cycle. Sample random keys among keys diff --git a/src/help.h b/src/help.h index edd15a3c9..b8b1efb95 100644 --- a/src/help.h +++ b/src/help.h @@ -459,12 +459,12 @@ struct commandHelp { 0, "1.2.0" }, { "FLUSHALL", - "[ASYNC]", + "[ASYNC|SYNC]", "Remove all keys from all databases", 9, "1.0.0" }, { "FLUSHDB", - "[ASYNC]", + "[ASYNC|SYNC]", "Remove all keys from the current database", 9, "1.0.0" }, @@ -518,6 +518,16 @@ struct commandHelp { "Returns the bit value at offset in the string value stored at key", 1, "2.2.0" }, + { "GETDEL", + "key", + "Get the value of a key and delete the key", + 1, + "6.2.0" }, + { "GETEX", + "key [EX seconds|PX milliseconds|EXAT timestamp|PXAT milliseconds-timestamp|PERSIST]", + "Get the value of a key and optionally set its expiration", + 1, + "6.2.0" }, { "GETRANGE", "key start end", "Get a substring of the string stored at a key", @@ -583,6 +593,11 @@ struct commandHelp { "Set multiple hash fields to multiple values", 5, "2.0.0" }, + { "HRANDFIELD", + "key [count [WITHVALUES]]", + "Get one or multiple random fields from a hash", + 5, + "6.2.0" }, { "HSCAN", "key cursor [MATCH pattern] [COUNT count]", "Incrementally iterate hash fields and associated values", @@ -989,7 +1004,7 @@ struct commandHelp { 10, "2.6.0" }, { "SCRIPT FLUSH", - "-", + "[ASYNC|SYNC]", "Remove all the scripts from the script cache.", 10, "2.6.0" }, @@ -1019,7 +1034,7 @@ struct commandHelp { 8, "1.0.0" }, { "SET", - "key value [EX seconds|PX milliseconds|KEEPTTL] [NX|XX] [GET]", + "key value [EX seconds|PX milliseconds|EXAT timestamp|PXAT milliseconds-timestamp|KEEPTTL] [NX|XX] [GET]", "Set the string value of a key", 1, "1.0.0" }, @@ -1323,6 +1338,11 @@ struct commandHelp { "Remove and return members with the lowest scores in a sorted set", 4, "5.0.0" }, + { "ZRANDMEMBER", + "key [count [WITHSCORES]]", + "Get one or multiple random elements from a sorted set", + 4, + "6.2.0" }, { "ZRANGE", "key min max [BYSCORE|BYLEX] [REV] [LIMIT offset count] [WITHSCORES]", "Return a range of members in a sorted set", diff --git a/src/lazyfree.c b/src/lazyfree.c index 8b9f0e2dc..f18b2027f 100644 --- a/src/lazyfree.c +++ b/src/lazyfree.c @@ -49,6 +49,14 @@ void lazyFreeTrackingTable(void *args[]) { atomicIncr(lazyfreed_objects,len); } +void lazyFreeLuaScripts(void *args[]) { + dict *lua_scripts = args[0]; + long long len = dictSize(lua_scripts); + dictRelease(lua_scripts); + atomicDecr(lazyfree_objects,len); + atomicIncr(lazyfreed_objects,len); +} + /* Return the number of currently pending objects to free. */ size_t lazyfreeGetPendingObjectsCount(void) { size_t aux; @@ -212,3 +220,13 @@ void freeTrackingRadixTreeAsync(rax *tracking) { atomicIncr(lazyfree_objects,tracking->numele); bioCreateLazyFreeJob(lazyFreeTrackingTable,1,tracking); } + +/* Free lua_scripts dict, if the dict is huge enough, free it in async way. */ +void freeLuaScriptsAsync(dict *lua_scripts) { + if (dictSize(lua_scripts) > LAZYFREE_THRESHOLD) { + atomicIncr(lazyfree_objects,dictSize(lua_scripts)); + bioCreateLazyFreeJob(lazyFreeLuaScripts,1,lua_scripts); + } else { + dictRelease(lua_scripts); + } +} diff --git a/src/module.c b/src/module.c index bf186f8b7..b04595801 100644 --- a/src/module.c +++ b/src/module.c @@ -29,7 +29,9 @@ #include "server.h" #include "cluster.h" +#include "slowlog.h" #include "rdb.h" +#include "monotonic.h" #include #include #include @@ -177,15 +179,25 @@ struct RedisModuleKey { void *iter; /* Iterator. */ int mode; /* Opening mode. */ - /* Zset iterator. */ - uint32_t ztype; /* REDISMODULE_ZSET_RANGE_* */ - zrangespec zrs; /* Score range. */ - zlexrangespec zlrs; /* Lex range. */ - uint32_t zstart; /* Start pos for positional ranges. */ - uint32_t zend; /* End pos for positional ranges. */ - void *zcurrent; /* Zset iterator current node. */ - int zer; /* Zset iterator end reached flag - (true if end was reached). */ + union { + struct { + /* Zset iterator, use only if value->type == OBJ_ZSET */ + uint32_t type; /* REDISMODULE_ZSET_RANGE_* */ + zrangespec rs; /* Score range. */ + zlexrangespec lrs; /* Lex range. */ + uint32_t start; /* Start pos for positional ranges. */ + uint32_t end; /* End pos for positional ranges. */ + void *current; /* Zset iterator current node. */ + int er; /* Zset iterator end reached flag + (true if end was reached). */ + } zset; + struct { + /* Stream, use only if value->type == OBJ_STREAM */ + streamID currentid; /* Current entry while iterating. */ + int64_t numfieldsleft; /* Fields left to fetch for current entry. */ + int signalready; /* Flag that signalKeyAsReady() is needed. */ + } stream; + } u; }; typedef struct RedisModuleKey RedisModuleKey; @@ -252,6 +264,9 @@ typedef struct RedisModuleBlockedClient { int dbid; /* Database number selected by the original client. */ int blocked_on_keys; /* If blocked via RM_BlockClientOnKeys(). */ int unblocked; /* Already on the moduleUnblocked list. */ + monotime background_timer; /* Timer tracking the start of background work */ + uint64_t background_duration; /* Current command background time duration. + Used for measuring latency of blocking cmds */ } RedisModuleBlockedClient; static pthread_mutex_t moduleUnblockedClientsMutex = PTHREAD_MUTEX_INITIALIZER; @@ -376,6 +391,7 @@ robj **moduleCreateArgvFromUserFormat(const char *cmdname, const char *fmt, int void moduleReplicateMultiIfNeeded(RedisModuleCtx *ctx); void RM_ZsetRangeStop(RedisModuleKey *kp); static void zsetKeyReset(RedisModuleKey *key); +static void moduleInitKeyTypeSpecific(RedisModuleKey *key); void RM_FreeDict(RedisModuleCtx *ctx, RedisModuleDict *d); void RM_FreeServerInfo(RedisModuleCtx *ctx, RedisModuleServerInfoData *data); @@ -478,17 +494,17 @@ void *RM_PoolAlloc(RedisModuleCtx *ctx, size_t bytes) { * Helpers for modules API implementation * -------------------------------------------------------------------------- */ -/* Create an empty key of the specified type. 'kp' must point to a key object - * opened for writing where the .value member is set to NULL because the +/* Create an empty key of the specified type. `key` must point to a key object + * opened for writing where the `.value` member is set to NULL because the * key was found to be non existing. * * On success REDISMODULE_OK is returned and the key is populated with * the value of the specified type. The function fails and returns * REDISMODULE_ERR if: * - * 1) The key is not open for writing. - * 2) The key is not empty. - * 3) The specified type is unknown. + * 1. The key is not open for writing. + * 2. The key is not empty. + * 3. The specified type is unknown. */ int moduleCreateEmptyKey(RedisModuleKey *key, int type) { robj *obj; @@ -509,10 +525,14 @@ int moduleCreateEmptyKey(RedisModuleKey *key, int type) { case REDISMODULE_KEYTYPE_HASH: obj = createHashObject(); break; + case REDISMODULE_KEYTYPE_STREAM: + obj = createStreamObject(); + break; default: return REDISMODULE_ERR; } dbAdd(key->db,key->key,obj); key->value = obj; + moduleInitKeyTypeSpecific(key); return REDISMODULE_OK; } @@ -900,6 +920,30 @@ long long RM_Milliseconds(void) { return mstime(); } +/* Mark a point in time that will be used as the start time to calculate + * the elapsed execution time when RM_BlockedClientMeasureTimeEnd() is called. + * Within the same command, you can call multiple times + * RM_BlockedClientMeasureTimeStart() and RM_BlockedClientMeasureTimeEnd() + * to accummulate indepedent time intervals to the background duration. + * This method always return REDISMODULE_OK. */ +int RM_BlockedClientMeasureTimeStart(RedisModuleBlockedClient *bc) { + elapsedStart(&(bc->background_timer)); + return REDISMODULE_OK; +} + +/* Mark a point in time that will be used as the end time + * to calculate the elapsed execution time. + * On success REDISMODULE_OK is returned. + * This method only returns REDISMODULE_ERR if no start time was + * previously defined ( meaning RM_BlockedClientMeasureTimeStart was not called ). */ +int RM_BlockedClientMeasureTimeEnd(RedisModuleBlockedClient *bc) { + // If the counter is 0 then we haven't called RM_BlockedClientMeasureTimeStart + if (!bc->background_timer) + return REDISMODULE_ERR; + bc->background_duration += elapsedUs(bc->background_timer); + return REDISMODULE_OK; +} + /* Set flags defining capabilities or behavior bit flags. * * REDISMODULE_OPTIONS_HANDLE_IO_ERRORS: @@ -933,9 +977,9 @@ int RM_SignalModifiedKey(RedisModuleCtx *ctx, RedisModuleString *keyname) { * keys, call replies and Redis string objects once the command returns. In most * cases this eliminates the need of calling the following functions: * - * 1) RedisModule_CloseKey() - * 2) RedisModule_FreeCallReply() - * 3) RedisModule_FreeString() + * 1. RedisModule_CloseKey() + * 2. RedisModule_FreeCallReply() + * 3. RedisModule_FreeString() * * These functions can still be used with automatic memory management enabled, * to optimize loops that make numerous allocations for example. */ @@ -1113,6 +1157,18 @@ RedisModuleString *RM_CreateStringFromString(RedisModuleCtx *ctx, const RedisMod return o; } +/* Creates a string from a stream ID. The returned string must be released with + * RedisModule_FreeString(), unless automatic memory is enabled. + * + * The passed context `ctx` may be NULL if necessary. See the + * RedisModule_CreateString() documentation for more info. */ +RedisModuleString *RM_CreateStringFromStreamID(RedisModuleCtx *ctx, const RedisModuleStreamID *id) { + streamID streamid = {id->ms, id->seq}; + RedisModuleString *o = createObjectFromStreamID(&streamid); + if (ctx != NULL) autoMemoryAdd(ctx, REDISMODULE_AM_STRING, o); + return o; +} + /* Free a module string object obtained with one of the Redis modules API calls * that return new string objects. * @@ -1139,9 +1195,9 @@ void RM_FreeString(RedisModuleCtx *ctx, RedisModuleString *str) { * Normally you want to call this function when, at the same time * the following conditions are true: * - * 1) You have automatic memory management enabled. - * 2) You want to create string objects. - * 3) Those string objects you create need to live *after* the callback + * 1. You have automatic memory management enabled. + * 2. You want to create string objects. + * 3. Those string objects you create need to live *after* the callback * function(for example a command implementation) creating them returns. * * Usually you want this in order to store the created string object @@ -1188,7 +1244,7 @@ void RM_RetainString(RedisModuleCtx *ctx, RedisModuleString *str) { * returned RedisModuleString. * * It is possible to call this function with a NULL context. - */ +*/ RedisModuleString* RM_HoldString(RedisModuleCtx *ctx, RedisModuleString *str) { if (str->refcount == OBJ_STATIC_REFCOUNT) { return RM_CreateStringFromString(ctx, str); @@ -1270,6 +1326,30 @@ int RM_StringToLongDouble(const RedisModuleString *str, long double *ld) { return retval ? REDISMODULE_OK : REDISMODULE_ERR; } +/* Convert the string into a stream ID, storing it at `*id`. + * Returns REDISMODULE_OK on success and returns REDISMODULE_ERR if the string + * is not a valid string representation of a stream ID. The special IDs "+" and + * "-" are allowed. + * + * RedisModuleStreamID is a struct with two 64-bit fields, which is used in + * stream functions and defined as + * + * typedef struct RedisModuleStreamID { + * uint64_t ms; + * uint64_t seq; + * } RedisModuleStreamID; + */ +int RM_StringToStreamID(const RedisModuleString *str, RedisModuleStreamID *id) { + streamID streamid; + if (streamParseID(str, &streamid) == C_OK) { + id->ms = streamid.ms; + id->seq = streamid.seq; + return REDISMODULE_OK; + } else { + return REDISMODULE_ERR; + } +} + /* Compare two string objects, returning -1, 0 or 1 respectively if * a < b, a == b, a > b. Strings are compared byte by byte as two * binary blobs without any encoding care / collation attempt. */ @@ -1322,7 +1402,7 @@ int RM_StringAppendBuffer(RedisModuleCtx *ctx, RedisModuleString *str, const cha * -------------------------------------------------------------------------- */ /* Send an error about the number of arguments given to the command, - * citing the command name in the error message. + * citing the command name in the error message. Returns REDISMODULE_OK. * * Example: * @@ -1394,7 +1474,7 @@ int RM_ReplyWithError(RedisModuleCtx *ctx, const char *err) { return REDISMODULE_OK; } -/* Reply with a simple string (+... \r\n in RESP protocol). This replies +/* Reply with a simple string (`+... \r\n` in RESP protocol). This replies * are suitable only when sending a small non-binary string with small * overhead, like "OK" or similar replies. * @@ -1742,7 +1822,7 @@ int RM_ReplicateVerbatim(RedisModuleCtx *ctx) { * 2. The ID increases monotonically. Clients connecting to the server later * are guaranteed to get IDs greater than any past ID previously seen. * - * Valid IDs are from 1 to 2^64-1. If 0 is returned it means there is no way + * Valid IDs are from 1 to 2^64 - 1. If 0 is returned it means there is no way * to fetch the ID in the context the function was currently called. * * After obtaining the ID, it is possible to check if the command execution @@ -2072,7 +2152,15 @@ static void moduleInitKey(RedisModuleKey *kp, RedisModuleCtx *ctx, robj *keyname kp->value = value; kp->iter = NULL; kp->mode = mode; - zsetKeyReset(kp); + if (kp->value) moduleInitKeyTypeSpecific(kp); +} + +/* Initialize the type-specific part of the key. Only when key has a value. */ +static void moduleInitKeyTypeSpecific(RedisModuleKey *key) { + switch (key->value->type) { + case OBJ_ZSET: zsetKeyReset(key); break; + case OBJ_STREAM: key->u.stream.signalready = 0; break; + } } /* Return an handle representing a Redis key, so that it is possible @@ -2115,8 +2203,13 @@ static void moduleCloseKey(RedisModuleKey *key) { int signal = SHOULD_SIGNAL_MODIFIED_KEYS(key->ctx); if ((key->mode & REDISMODULE_WRITE) && signal) signalModifiedKey(key->ctx->client,key->db,key->key); - /* TODO: if (key->iter) RM_KeyIteratorStop(kp); */ + if (key->iter) zfree(key->iter); RM_ZsetRangeStop(key); + if (key && key->value && key->value->type == OBJ_STREAM && + key->u.stream.signalready) { + /* One of more RM_StreamAdd() have been done. */ + signalKeyAsReady(key->db, key->key, OBJ_STREAM); + } decrRefCount(key->key); } @@ -2376,9 +2469,10 @@ int RM_ListPush(RedisModuleKey *key, int where, RedisModuleString *ele) { * that the user should be free with RM_FreeString() or by enabling * automatic memory. 'where' specifies if the element should be popped from * head or tail. The command returns NULL if: - * 1) The list is empty. - * 2) The key was not open for writing. - * 3) The key is not a list. */ + * + * 1. The list is empty. + * 2. The key was not open for writing. + * 3. The key is not a list. */ RedisModuleString *RM_ListPop(RedisModuleKey *key, int where) { if (!(key->mode & REDISMODULE_WRITE) || key->value == NULL || @@ -2398,7 +2492,7 @@ RedisModuleString *RM_ListPop(RedisModuleKey *key, int where) { /* Conversion from/to public flags of the Modules API and our private flags, * so that we have everything decoupled. */ -int RM_ZsetAddFlagsToCoreFlags(int flags) { +int moduleZsetAddFlagsToCoreFlags(int flags) { int retflags = 0; if (flags & REDISMODULE_ZADD_XX) retflags |= ZADD_XX; if (flags & REDISMODULE_ZADD_NX) retflags |= ZADD_NX; @@ -2408,7 +2502,7 @@ int RM_ZsetAddFlagsToCoreFlags(int flags) { } /* See previous function comment. */ -int RM_ZsetAddFlagsFromCoreFlags(int flags) { +int moduleZsetAddFlagsFromCoreFlags(int flags) { int retflags = 0; if (flags & ZADD_ADDED) retflags |= REDISMODULE_ZADD_ADDED; if (flags & ZADD_UPDATED) retflags |= REDISMODULE_ZADD_UPDATED; @@ -2453,12 +2547,12 @@ int RM_ZsetAdd(RedisModuleKey *key, double score, RedisModuleString *ele, int *f if (!(key->mode & REDISMODULE_WRITE)) return REDISMODULE_ERR; if (key->value && key->value->type != OBJ_ZSET) return REDISMODULE_ERR; if (key->value == NULL) moduleCreateEmptyKey(key,REDISMODULE_KEYTYPE_ZSET); - if (flagsptr) flags = RM_ZsetAddFlagsToCoreFlags(*flagsptr); + if (flagsptr) flags = moduleZsetAddFlagsToCoreFlags(*flagsptr); if (zsetAdd(key->value,score,ele->ptr,&flags,NULL) == 0) { if (flagsptr) *flagsptr = 0; return REDISMODULE_ERR; } - if (flagsptr) *flagsptr = RM_ZsetAddFlagsFromCoreFlags(flags); + if (flagsptr) *flagsptr = moduleZsetAddFlagsFromCoreFlags(flags); return REDISMODULE_OK; } @@ -2480,7 +2574,7 @@ int RM_ZsetIncrby(RedisModuleKey *key, double score, RedisModuleString *ele, int if (!(key->mode & REDISMODULE_WRITE)) return REDISMODULE_ERR; if (key->value && key->value->type != OBJ_ZSET) return REDISMODULE_ERR; if (key->value == NULL) moduleCreateEmptyKey(key,REDISMODULE_KEYTYPE_ZSET); - if (flagsptr) flags = RM_ZsetAddFlagsToCoreFlags(*flagsptr); + if (flagsptr) flags = moduleZsetAddFlagsToCoreFlags(*flagsptr); flags |= ZADD_INCR; if (zsetAdd(key->value,score,ele->ptr,&flags,newscore) == 0) { if (flagsptr) *flagsptr = 0; @@ -2491,7 +2585,7 @@ int RM_ZsetIncrby(RedisModuleKey *key, double score, RedisModuleString *ele, int *flagsptr = 0; return REDISMODULE_ERR; } - if (flagsptr) *flagsptr = RM_ZsetAddFlagsFromCoreFlags(flags); + if (flagsptr) *flagsptr = moduleZsetAddFlagsFromCoreFlags(flags); return REDISMODULE_OK; } @@ -2544,16 +2638,17 @@ int RM_ZsetScore(RedisModuleKey *key, RedisModuleString *ele, double *score) { * -------------------------------------------------------------------------- */ void zsetKeyReset(RedisModuleKey *key) { - key->ztype = REDISMODULE_ZSET_RANGE_NONE; - key->zcurrent = NULL; - key->zer = 1; + key->u.zset.type = REDISMODULE_ZSET_RANGE_NONE; + key->u.zset.current = NULL; + key->u.zset.er = 1; } /* Stop a sorted set iteration. */ void RM_ZsetRangeStop(RedisModuleKey *key) { + if (!key->value || key->value->type != OBJ_ZSET) return; /* Free resources if needed. */ - if (key->ztype == REDISMODULE_ZSET_RANGE_LEX) - zslFreeLexRange(&key->zlrs); + if (key->u.zset.type == REDISMODULE_ZSET_RANGE_LEX) + zslFreeLexRange(&key->u.zset.lrs); /* Setup sensible values so that misused iteration API calls when an * iterator is not active will result into something more sensible * than crashing. */ @@ -2562,7 +2657,7 @@ void RM_ZsetRangeStop(RedisModuleKey *key) { /* Return the "End of range" flag value to signal the end of the iteration. */ int RM_ZsetRangeEndReached(RedisModuleKey *key) { - return key->zer; + return key->u.zset.er; } /* Helper function for RM_ZsetFirstInScoreRange() and RM_ZsetLastInScoreRange(). @@ -2575,29 +2670,29 @@ int zsetInitScoreRange(RedisModuleKey *key, double min, double max, int minex, i if (!key->value || key->value->type != OBJ_ZSET) return REDISMODULE_ERR; RM_ZsetRangeStop(key); - key->ztype = REDISMODULE_ZSET_RANGE_SCORE; - key->zer = 0; + key->u.zset.type = REDISMODULE_ZSET_RANGE_SCORE; + key->u.zset.er = 0; /* Setup the range structure used by the sorted set core implementation * in order to seek at the specified element. */ - zrangespec *zrs = &key->zrs; + zrangespec *zrs = &key->u.zset.rs; zrs->min = min; zrs->max = max; zrs->minex = minex; zrs->maxex = maxex; if (key->value->encoding == OBJ_ENCODING_ZIPLIST) { - key->zcurrent = first ? zzlFirstInRange(key->value->ptr,zrs) : - zzlLastInRange(key->value->ptr,zrs); + key->u.zset.current = first ? zzlFirstInRange(key->value->ptr,zrs) : + zzlLastInRange(key->value->ptr,zrs); } else if (key->value->encoding == OBJ_ENCODING_SKIPLIST) { zset *zs = key->value->ptr; zskiplist *zsl = zs->zsl; - key->zcurrent = first ? zslFirstInRange(zsl,zrs) : - zslLastInRange(zsl,zrs); + key->u.zset.current = first ? zslFirstInRange(zsl,zrs) : + zslLastInRange(zsl,zrs); } else { serverPanic("Unsupported zset encoding"); } - if (key->zcurrent == NULL) key->zer = 1; + if (key->u.zset.current == NULL) key->u.zset.er = 1; return REDISMODULE_OK; } @@ -2610,8 +2705,8 @@ int zsetInitScoreRange(RedisModuleKey *key, double min, double max, int minex, i * The range is specified according to the two double values 'min' and 'max'. * Both can be infinite using the following two macros: * - * REDISMODULE_POSITIVE_INFINITE for positive infinite value - * REDISMODULE_NEGATIVE_INFINITE for negative infinite value + * * REDISMODULE_POSITIVE_INFINITE for positive infinite value + * * REDISMODULE_NEGATIVE_INFINITE for negative infinite value * * 'minex' and 'maxex' parameters, if true, respectively setup a range * where the min and max value are exclusive (not included) instead of @@ -2639,29 +2734,29 @@ int zsetInitLexRange(RedisModuleKey *key, RedisModuleString *min, RedisModuleStr if (!key->value || key->value->type != OBJ_ZSET) return REDISMODULE_ERR; RM_ZsetRangeStop(key); - key->zer = 0; + key->u.zset.er = 0; /* Setup the range structure used by the sorted set core implementation * in order to seek at the specified element. */ - zlexrangespec *zlrs = &key->zlrs; + zlexrangespec *zlrs = &key->u.zset.lrs; if (zslParseLexRange(min, max, zlrs) == C_ERR) return REDISMODULE_ERR; /* Set the range type to lex only after successfully parsing the range, * otherwise we don't want the zlexrangespec to be freed. */ - key->ztype = REDISMODULE_ZSET_RANGE_LEX; + key->u.zset.type = REDISMODULE_ZSET_RANGE_LEX; if (key->value->encoding == OBJ_ENCODING_ZIPLIST) { - key->zcurrent = first ? zzlFirstInLexRange(key->value->ptr,zlrs) : - zzlLastInLexRange(key->value->ptr,zlrs); + key->u.zset.current = first ? zzlFirstInLexRange(key->value->ptr,zlrs) : + zzlLastInLexRange(key->value->ptr,zlrs); } else if (key->value->encoding == OBJ_ENCODING_SKIPLIST) { zset *zs = key->value->ptr; zskiplist *zsl = zs->zsl; - key->zcurrent = first ? zslFirstInLexRange(zsl,zlrs) : - zslLastInLexRange(zsl,zlrs); + key->u.zset.current = first ? zslFirstInLexRange(zsl,zlrs) : + zslLastInLexRange(zsl,zlrs); } else { serverPanic("Unsupported zset encoding"); } - if (key->zcurrent == NULL) key->zer = 1; + if (key->u.zset.current == NULL) key->u.zset.er = 1; return REDISMODULE_OK; } @@ -2694,10 +2789,11 @@ int RM_ZsetLastInLexRange(RedisModuleKey *key, RedisModuleString *min, RedisModu RedisModuleString *RM_ZsetRangeCurrentElement(RedisModuleKey *key, double *score) { RedisModuleString *str; - if (key->zcurrent == NULL) return NULL; + if (!key->value || key->value->type != OBJ_ZSET) return NULL; + if (key->u.zset.current == NULL) return NULL; if (key->value->encoding == OBJ_ENCODING_ZIPLIST) { unsigned char *eptr, *sptr; - eptr = key->zcurrent; + eptr = key->u.zset.current; sds ele = ziplistGetObject(eptr); if (score) { sptr = ziplistNext(key->value->ptr,eptr); @@ -2705,7 +2801,7 @@ RedisModuleString *RM_ZsetRangeCurrentElement(RedisModuleKey *key, double *score } str = createObject(OBJ_STRING,ele); } else if (key->value->encoding == OBJ_ENCODING_SKIPLIST) { - zskiplistNode *ln = key->zcurrent; + zskiplistNode *ln = key->u.zset.current; if (score) *score = ln->score; str = createStringObject(ln->ele,sdslen(ln->ele)); } else { @@ -2719,58 +2815,59 @@ RedisModuleString *RM_ZsetRangeCurrentElement(RedisModuleKey *key, double *score * a next element, 0 if we are already at the latest element or the range * does not include any item at all. */ int RM_ZsetRangeNext(RedisModuleKey *key) { - if (!key->ztype || !key->zcurrent) return 0; /* No active iterator. */ + if (!key->value || key->value->type != OBJ_ZSET) return 0; + if (!key->u.zset.type || !key->u.zset.current) return 0; /* No active iterator. */ if (key->value->encoding == OBJ_ENCODING_ZIPLIST) { unsigned char *zl = key->value->ptr; - unsigned char *eptr = key->zcurrent; + unsigned char *eptr = key->u.zset.current; unsigned char *next; next = ziplistNext(zl,eptr); /* Skip element. */ if (next) next = ziplistNext(zl,next); /* Skip score. */ if (next == NULL) { - key->zer = 1; + key->u.zset.er = 1; return 0; } else { /* Are we still within the range? */ - if (key->ztype == REDISMODULE_ZSET_RANGE_SCORE) { + if (key->u.zset.type == REDISMODULE_ZSET_RANGE_SCORE) { /* Fetch the next element score for the * range check. */ unsigned char *saved_next = next; next = ziplistNext(zl,next); /* Skip next element. */ double score = zzlGetScore(next); /* Obtain the next score. */ - if (!zslValueLteMax(score,&key->zrs)) { - key->zer = 1; + if (!zslValueLteMax(score,&key->u.zset.rs)) { + key->u.zset.er = 1; return 0; } next = saved_next; - } else if (key->ztype == REDISMODULE_ZSET_RANGE_LEX) { - if (!zzlLexValueLteMax(next,&key->zlrs)) { - key->zer = 1; + } else if (key->u.zset.type == REDISMODULE_ZSET_RANGE_LEX) { + if (!zzlLexValueLteMax(next,&key->u.zset.lrs)) { + key->u.zset.er = 1; return 0; } } - key->zcurrent = next; + key->u.zset.current = next; return 1; } } else if (key->value->encoding == OBJ_ENCODING_SKIPLIST) { - zskiplistNode *ln = key->zcurrent, *next = ln->level[0].forward; + zskiplistNode *ln = key->u.zset.current, *next = ln->level[0].forward; if (next == NULL) { - key->zer = 1; + key->u.zset.er = 1; return 0; } else { /* Are we still within the range? */ - if (key->ztype == REDISMODULE_ZSET_RANGE_SCORE && - !zslValueLteMax(next->score,&key->zrs)) + if (key->u.zset.type == REDISMODULE_ZSET_RANGE_SCORE && + !zslValueLteMax(next->score,&key->u.zset.rs)) { - key->zer = 1; + key->u.zset.er = 1; return 0; - } else if (key->ztype == REDISMODULE_ZSET_RANGE_LEX) { - if (!zslLexValueLteMax(next->ele,&key->zlrs)) { - key->zer = 1; + } else if (key->u.zset.type == REDISMODULE_ZSET_RANGE_LEX) { + if (!zslLexValueLteMax(next->ele,&key->u.zset.lrs)) { + key->u.zset.er = 1; return 0; } } - key->zcurrent = next; + key->u.zset.current = next; return 1; } } else { @@ -2782,58 +2879,59 @@ int RM_ZsetRangeNext(RedisModuleKey *key) { * a previous element, 0 if we are already at the first element or the range * does not include any item at all. */ int RM_ZsetRangePrev(RedisModuleKey *key) { - if (!key->ztype || !key->zcurrent) return 0; /* No active iterator. */ + if (!key->value || key->value->type != OBJ_ZSET) return 0; + if (!key->u.zset.type || !key->u.zset.current) return 0; /* No active iterator. */ if (key->value->encoding == OBJ_ENCODING_ZIPLIST) { unsigned char *zl = key->value->ptr; - unsigned char *eptr = key->zcurrent; + unsigned char *eptr = key->u.zset.current; unsigned char *prev; prev = ziplistPrev(zl,eptr); /* Go back to previous score. */ if (prev) prev = ziplistPrev(zl,prev); /* Back to previous ele. */ if (prev == NULL) { - key->zer = 1; + key->u.zset.er = 1; return 0; } else { /* Are we still within the range? */ - if (key->ztype == REDISMODULE_ZSET_RANGE_SCORE) { + if (key->u.zset.type == REDISMODULE_ZSET_RANGE_SCORE) { /* Fetch the previous element score for the * range check. */ unsigned char *saved_prev = prev; prev = ziplistNext(zl,prev); /* Skip element to get the score.*/ double score = zzlGetScore(prev); /* Obtain the prev score. */ - if (!zslValueGteMin(score,&key->zrs)) { - key->zer = 1; + if (!zslValueGteMin(score,&key->u.zset.rs)) { + key->u.zset.er = 1; return 0; } prev = saved_prev; - } else if (key->ztype == REDISMODULE_ZSET_RANGE_LEX) { - if (!zzlLexValueGteMin(prev,&key->zlrs)) { - key->zer = 1; + } else if (key->u.zset.type == REDISMODULE_ZSET_RANGE_LEX) { + if (!zzlLexValueGteMin(prev,&key->u.zset.lrs)) { + key->u.zset.er = 1; return 0; } } - key->zcurrent = prev; + key->u.zset.current = prev; return 1; } } else if (key->value->encoding == OBJ_ENCODING_SKIPLIST) { - zskiplistNode *ln = key->zcurrent, *prev = ln->backward; + zskiplistNode *ln = key->u.zset.current, *prev = ln->backward; if (prev == NULL) { - key->zer = 1; + key->u.zset.er = 1; return 0; } else { /* Are we still within the range? */ - if (key->ztype == REDISMODULE_ZSET_RANGE_SCORE && - !zslValueGteMin(prev->score,&key->zrs)) + if (key->u.zset.type == REDISMODULE_ZSET_RANGE_SCORE && + !zslValueGteMin(prev->score,&key->u.zset.rs)) { - key->zer = 1; + key->u.zset.er = 1; return 0; - } else if (key->ztype == REDISMODULE_ZSET_RANGE_LEX) { - if (!zslLexValueGteMin(prev->ele,&key->zlrs)) { - key->zer = 1; + } else if (key->u.zset.type == REDISMODULE_ZSET_RANGE_LEX) { + if (!zslLexValueGteMin(prev->ele,&key->u.zset.lrs)) { + key->u.zset.er = 1; return 0; } } - key->zcurrent = prev; + key->u.zset.current = prev; return 1; } } else { @@ -2970,7 +3068,7 @@ int RM_HashSet(RedisModuleKey *key, int flags, ...) { * * RedisModuleString *first, *second; * RedisModule_HashGet(mykey,REDISMODULE_HASH_NONE,argv[1],&first, - * argv[2],&second,NULL); + * argv[2],&second,NULL); * * As with RedisModule_HashSet() the behavior of the command can be specified * passing flags different than REDISMODULE_HASH_NONE: @@ -3048,6 +3146,455 @@ int RM_HashGet(RedisModuleKey *key, int flags, ...) { return REDISMODULE_OK; } +/* -------------------------------------------------------------------------- + * Key API for the stream type. + * -------------------------------------------------------------------------- */ + +/* Adds an entry to a stream. Like XADD without trimming. + * + * - `key`: The key where the stream is (or will be) stored + * - `flags`: A bit field of + * - `REDISMODULE_STREAM_ADD_AUTOID`: Assign a stream ID automatically, like + * `*` in the XADD command. + * - `id`: If the `AUTOID` flag is set, this is where the assigned ID is + * returned. Can be NULL if `AUTOID` is set, if you don't care to receive the + * ID. If `AUTOID` is not set, this is the requested ID. + * - `argv`: A pointer to an array of size `numfields * 2` containing the + * fields and values. + * - `numfields`: The number of field-value pairs in `argv`. + * + * Returns REDISMODULE_OK if an entry has been added. On failure, + * REDISMODULE_ERR is returned and `errno` is set as follows: + * + * - EINVAL if called with invalid arguments + * - ENOTSUP if the key refers to a value of a type other than stream + * - EBADF if the key was not opened for writing + * - EDOM if the given ID was 0-0 or not greater than all other IDs in the + * stream (only if the AUTOID flag is unset) + * - EFBIG if the stream has reached the last possible ID + */ +int RM_StreamAdd(RedisModuleKey *key, int flags, RedisModuleStreamID *id, RedisModuleString **argv, long numfields) { + /* Validate args */ + if (!key || (numfields != 0 && !argv) || /* invalid key or argv */ + (flags & ~(REDISMODULE_STREAM_ADD_AUTOID)) || /* invalid flags */ + (!(flags & REDISMODULE_STREAM_ADD_AUTOID) && !id)) { /* id required */ + errno = EINVAL; + return REDISMODULE_ERR; + } else if (key->value && key->value->type != OBJ_STREAM) { + errno = ENOTSUP; /* wrong type */ + return REDISMODULE_ERR; + } else if (!(key->mode & REDISMODULE_WRITE)) { + errno = EBADF; /* key not open for writing */ + return REDISMODULE_ERR; + } else if (!(flags & REDISMODULE_STREAM_ADD_AUTOID) && + id->ms == 0 && id->seq == 0) { + errno = EDOM; /* ID out of range */ + return REDISMODULE_ERR; + } + + /* Create key if necessery */ + int created = 0; + if (key->value == NULL) { + moduleCreateEmptyKey(key, REDISMODULE_KEYTYPE_STREAM); + created = 1; + } + + stream *s = key->value->ptr; + if (s->last_id.ms == UINT64_MAX && s->last_id.seq == UINT64_MAX) { + /* The stream has reached the last possible ID */ + errno = EFBIG; + return REDISMODULE_ERR; + } + + streamID added_id; + streamID use_id; + streamID *use_id_ptr = NULL; + if (!(flags & REDISMODULE_STREAM_ADD_AUTOID)) { + use_id.ms = id->ms; + use_id.seq = id->seq; + use_id_ptr = &use_id; + } + if (streamAppendItem(s, argv, numfields, &added_id, use_id_ptr) == C_ERR) { + /* ID not greater than all existing IDs in the stream */ + errno = EDOM; + return REDISMODULE_ERR; + } + /* Postponed signalKeyAsReady(). Done implicitly by moduleCreateEmptyKey() + * so not needed if the stream has just been created. */ + if (!created) key->u.stream.signalready = 1; + + if (id != NULL) { + id->ms = added_id.ms; + id->seq = added_id.seq; + } + + return REDISMODULE_OK; +} + +/* Deletes an entry from a stream. + * + * - `key`: A key opened for writing, with no stream iterator started. + * - `id`: The stream ID of the entry to delete. + * + * Returns REDISMODULE_OK on success. On failure, REDISMODULE_ERR is returned + * and `errno` is set as follows: + * + * - EINVAL if called with invalid arguments + * - ENOTSUP if the key refers to a value of a type other than stream or if the + * key is empty + * - EBADF if the key was not opened for writing or if a stream iterator is + * associated with the key + * - ENOENT if no entry with the given stream ID exists + * + * See also RM_StreamIteratorDelete() for deleting the current entry while + * iterating using a stream iterator. + */ +int RM_StreamDelete(RedisModuleKey *key, RedisModuleStreamID *id) { + if (!key || !id) { + errno = EINVAL; + return REDISMODULE_ERR; + } else if (!key->value || key->value->type != OBJ_STREAM) { + errno = ENOTSUP; /* wrong type */ + return REDISMODULE_ERR; + } else if (!(key->mode & REDISMODULE_WRITE) || + key->iter != NULL) { + errno = EBADF; /* key not opened for writing or iterator started */ + return REDISMODULE_ERR; + } + stream *s = key->value->ptr; + streamID streamid = {id->ms, id->seq}; + if (streamDeleteItem(s, &streamid)) { + return REDISMODULE_OK; + } else { + errno = ENOENT; /* no entry with this id */ + return REDISMODULE_ERR; + } +} + +/* Sets up a stream iterator. + * + * - `key`: The stream key opened for reading using RedisModule_OpenKey(). + * - `flags`: + * - `REDISMODULE_STREAM_ITERATOR_EXCLUSIVE`: Don't include `start` and `end` + * in the iterated range. + * - `REDISMODULE_STREAM_ITERATOR_REVERSE`: Iterate in reverse order, starting + * from the `end` of the range. + * - `start`: The lower bound of the range. Use NULL for the beginning of the + * stream. + * - `end`: The upper bound of the range. Use NULL for the end of the stream. + * + * Returns REDISMODULE_OK on success. On failure, REDISMODULE_ERR is returned + * and `errno` is set as follows: + * + * - EINVAL if called with invalid arguments + * - ENOTSUP if the key refers to a value of a type other than stream or if the + * key is empty + * - EBADF if the key was not opened for writing or if a stream iterator is + * already associated with the key + * - EDOM if `start` or `end` is outside the valid range + * + * Returns REDISMODULE_OK on success and REDISMODULE_ERR if the key doesn't + * refer to a stream or if invalid arguments were given. + * + * The stream IDs are retrieved using RedisModule_StreamIteratorNextID() and + * for each stream ID, the fields and values are retrieved using + * RedisModule_StreamIteratorNextField(). The iterator is freed by calling + * RedisModule_StreamIteratorStop(). + * + * Example (error handling omitted): + * + * RedisModule_StreamIteratorStart(key, 0, startid_ptr, endid_ptr); + * RedisModuleStreamID id; + * long numfields; + * while (RedisModule_StreamIteratorNextID(key, &id, &numfields) == + * REDISMODULE_OK) { + * RedisModuleString *field, *value; + * while (RedisModule_StreamIteratorNextField(key, &field, &value) == + * REDISMODULE_OK) { + * // + * // ... Do stuff ... + * // + * RedisModule_Free(field); + * RedisModule_Free(value); + * } + * } + * RedisModule_StreamIteratorStop(key); + */ +int RM_StreamIteratorStart(RedisModuleKey *key, int flags, RedisModuleStreamID *start, RedisModuleStreamID *end) { + /* check args */ + if (!key || + (flags & ~(REDISMODULE_STREAM_ITERATOR_EXCLUSIVE | + REDISMODULE_STREAM_ITERATOR_REVERSE))) { + errno = EINVAL; /* key missing or invalid flags */ + return REDISMODULE_ERR; + } else if (!key->value || key->value->type != OBJ_STREAM) { + errno = ENOTSUP; + return REDISMODULE_ERR; /* not a stream */ + } else if (key->iter) { + errno = EBADF; /* iterator already started */ + return REDISMODULE_ERR; + } + + /* define range for streamIteratorStart() */ + streamID lower, upper; + if (start) lower = (streamID){start->ms, start->seq}; + if (end) upper = (streamID){end->ms, end->seq}; + if (flags & REDISMODULE_STREAM_ITERATOR_EXCLUSIVE) { + if ((start && streamIncrID(&lower) != C_OK) || + (end && streamDecrID(&upper) != C_OK)) { + errno = EDOM; /* end is 0-0 or start is MAX-MAX? */ + return REDISMODULE_ERR; + } + } + + /* create iterator */ + stream *s = key->value->ptr; + int rev = flags & REDISMODULE_STREAM_ITERATOR_REVERSE; + streamIterator *si = zmalloc(sizeof(*si)); + streamIteratorStart(si, s, start ? &lower : NULL, end ? &upper : NULL, rev); + key->iter = si; + key->u.stream.currentid.ms = 0; /* for RM_StreamIteratorDelete() */ + key->u.stream.currentid.seq = 0; + key->u.stream.numfieldsleft = 0; /* for RM_StreamIteratorNextField() */ + return REDISMODULE_OK; +} + +/* Stops a stream iterator created using RedisModule_StreamIteratorStart() and + * reclaims its memory. + * + * Returns REDISMODULE_OK on success. On failure, REDISMODULE_ERR is returned + * and `errno` is set as follows: + * + * - EINVAL if called with a NULL key + * - ENOTSUP if the key refers to a value of a type other than stream or if the + * key is empty + * - EBADF if the key was not opened for writing or if no stream iterator is + * associated with the key + */ +int RM_StreamIteratorStop(RedisModuleKey *key) { + if (!key) { + errno = EINVAL; + return REDISMODULE_ERR; + } else if (!key->value || key->value->type != OBJ_STREAM) { + errno = ENOTSUP; + return REDISMODULE_ERR; + } else if (!key->iter) { + errno = EBADF; + return REDISMODULE_ERR; + } + zfree(key->iter); + key->iter = NULL; + return REDISMODULE_OK; +} + +/* Finds the next stream entry and returns its stream ID and the number of + * fields. + * + * - `key`: Key for which a stream iterator has been started using + * RedisModule_StreamIteratorStart(). + * - `id`: The stream ID returned. NULL if you don't care. + * - `numfields`: The number of fields in the found stream entry. NULL if you + * don't care. + * + * Returns REDISMODULE_OK and sets `*id` and `*numfields` if an entry was found. + * On failure, REDISMODULE_ERR is returned and `errno` is set as follows: + * + * - EINVAL if called with a NULL key + * - ENOTSUP if the key refers to a value of a type other than stream or if the + * key is empty + * - EBADF if no stream iterator is associated with the key + * - ENOENT if there are no more entries in the range of the iterator + * + * In practice, if RM_StreamIteratorNextID() is called after a successful call + * to RM_StreamIteratorStart() and with the same key, it is safe to assume that + * an REDISMODULE_ERR return value means that there are no more entries. + * + * Use RedisModule_StreamIteratorNextField() to retrieve the fields and values. + * See the example at RedisModule_StreamIteratorStart(). + */ +int RM_StreamIteratorNextID(RedisModuleKey *key, RedisModuleStreamID *id, long *numfields) { + if (!key) { + errno = EINVAL; + return REDISMODULE_ERR; + } else if (!key->value || key->value->type != OBJ_STREAM) { + errno = ENOTSUP; + return REDISMODULE_ERR; + } else if (!key->iter) { + errno = EBADF; + return REDISMODULE_ERR; + } + streamIterator *si = key->iter; + int64_t *num_ptr = &key->u.stream.numfieldsleft; + streamID *streamid_ptr = &key->u.stream.currentid; + if (streamIteratorGetID(si, streamid_ptr, num_ptr)) { + if (id) { + id->ms = streamid_ptr->ms; + id->seq = streamid_ptr->seq; + } + if (numfields) *numfields = *num_ptr; + return REDISMODULE_OK; + } else { + /* No entry found. */ + key->u.stream.currentid.ms = 0; /* for RM_StreamIteratorDelete() */ + key->u.stream.currentid.seq = 0; + key->u.stream.numfieldsleft = 0; /* for RM_StreamIteratorNextField() */ + errno = ENOENT; + return REDISMODULE_ERR; + } +} + +/* Retrieves the next field of the current stream ID and its corresponding value + * in a stream iteration. This function should be called repeatedly after calling + * RedisModule_StreamIteratorNextID() to fetch each field-value pair. + * + * - `key`: Key where a stream iterator has been started. + * - `field_ptr`: This is where the field is returned. + * - `value_ptr`: This is where the value is returned. + * + * Returns REDISMODULE_OK and points `*field_ptr` and `*value_ptr` to freshly + * allocated RedisModuleString objects. The string objects are freed + * automatically when the callback finishes if automatic memory is enabled. On + * failure, REDISMODULE_ERR is returned and `errno` is set as follows: + * + * - EINVAL if called with a NULL key + * - ENOTSUP if the key refers to a value of a type other than stream or if the + * key is empty + * - EBADF if no stream iterator is associated with the key + * - ENOENT if there are no more fields in the current stream entry + * + * In practice, if RM_StreamIteratorNextField() is called after a successful + * call to RM_StreamIteratorNextID() and with the same key, it is safe to assume + * that an REDISMODULE_ERR return value means that there are no more fields. + * + * See the example at RedisModule_StreamIteratorStart(). + */ +int RM_StreamIteratorNextField(RedisModuleKey *key, RedisModuleString **field_ptr, RedisModuleString **value_ptr) { + if (!key) { + errno = EINVAL; + return REDISMODULE_ERR; + } else if (!key->value || key->value->type != OBJ_STREAM) { + errno = ENOTSUP; + return REDISMODULE_ERR; + } else if (!key->iter) { + errno = EBADF; + return REDISMODULE_ERR; + } else if (key->u.stream.numfieldsleft <= 0) { + errno = ENOENT; + return REDISMODULE_ERR; + } + streamIterator *si = key->iter; + unsigned char *field, *value; + int64_t field_len, value_len; + streamIteratorGetField(si, &field, &value, &field_len, &value_len); + if (field_ptr) { + *field_ptr = createRawStringObject((char *)field, field_len); + autoMemoryAdd(key->ctx, REDISMODULE_AM_STRING, *field_ptr); + } + if (value_ptr) { + *value_ptr = createRawStringObject((char *)value, value_len); + autoMemoryAdd(key->ctx, REDISMODULE_AM_STRING, *value_ptr); + } + key->u.stream.numfieldsleft--; + return REDISMODULE_OK; +} + +/* Deletes the current stream entry while iterating. + * + * This function can be called after RM_StreamIteratorNextID() or after any + * calls to RM_StreamIteratorNextField(). + * + * Returns REDISMODULE_OK on success. On failure, REDISMODULE_ERR is returned + * and `errno` is set as follows: + * + * - EINVAL if key is NULL + * - ENOTSUP if the key is empty or is of another type than stream + * - EBADF if the key is not opened for writing, if no iterator has been started + * - ENOENT if the iterator has no current stream entry + */ +int RM_StreamIteratorDelete(RedisModuleKey *key) { + if (!key) { + errno = EINVAL; + return REDISMODULE_ERR; + } else if (!key->value || key->value->type != OBJ_STREAM) { + errno = ENOTSUP; + return REDISMODULE_ERR; + } else if (!(key->mode & REDISMODULE_WRITE) || !key->iter) { + errno = EBADF; + return REDISMODULE_ERR; + } else if (key->u.stream.currentid.ms == 0 && + key->u.stream.currentid.seq == 0) { + errno = ENOENT; + return REDISMODULE_ERR; + } + streamIterator *si = key->iter; + streamIteratorRemoveEntry(si, &key->u.stream.currentid); + key->u.stream.currentid.ms = 0; /* Make sure repeated Delete() fails */ + key->u.stream.currentid.seq = 0; + key->u.stream.numfieldsleft = 0; /* Make sure NextField() fails */ + return REDISMODULE_OK; +} + +/* Trim a stream by length, similar to XTRIM with MAXLEN. + * + * - `key`: Key opened for writing. + * - `flags`: A bitfield of + * - `REDISMODULE_STREAM_TRIM_APPROX`: Trim less if it improves performance, + * like XTRIM with `~`. + * - `length`: The number of stream entries to keep after trimming. + * + * Returns the number of entries deleted. On failure, a negative value is + * returned and `errno` is set as follows: + * + * - EINVAL if called with invalid arguments + * - ENOTSUP if the key is empty or of a type other than stream + * - EBADF if the key is not opened for writing + */ +long long RM_StreamTrimByLength(RedisModuleKey *key, int flags, long long length) { + if (!key || (flags & ~(REDISMODULE_STREAM_TRIM_APPROX)) || length < 0) { + errno = EINVAL; + return -1; + } else if (!key->value || key->value->type != OBJ_STREAM) { + errno = ENOTSUP; + return -1; + } else if (!(key->mode & REDISMODULE_WRITE)) { + errno = EBADF; + return -1; + } + int approx = flags & REDISMODULE_STREAM_TRIM_APPROX ? 1 : 0; + return streamTrimByLength((stream *)key->value->ptr, length, approx); +} + +/* Trim a stream by ID, similar to XTRIM with MINID. + * + * - `key`: Key opened for writing. + * - `flags`: A bitfield of + * - `REDISMODULE_STREAM_TRIM_APPROX`: Trim less if it improves performance, + * like XTRIM with `~`. + * - `id`: The smallest stream ID to keep after trimming. + * + * Returns the number of entries deleted. On failure, a negative value is + * returned and `errno` is set as follows: + * + * - EINVAL if called with invalid arguments + * - ENOTSUP if the key is empty or of a type other than stream + * - EBADF if the key is not opened for writing + */ +long long RM_StreamTrimByID(RedisModuleKey *key, int flags, RedisModuleStreamID *id) { + if (!key || (flags & ~(REDISMODULE_STREAM_TRIM_APPROX)) || !id) { + errno = EINVAL; + return -1; + } else if (!key->value || key->value->type != OBJ_STREAM) { + errno = ENOTSUP; + return -1; + } else if (!(key->mode & REDISMODULE_WRITE)) { + errno = EBADF; + return -1; + } + int approx = flags & REDISMODULE_STREAM_TRIM_APPROX ? 1 : 0; + streamID minid = (streamID){id->ms, id->seq}; + return streamTrimByID((stream *)key->value->ptr, minid, approx); +} + /* -------------------------------------------------------------------------- * Redis <-> Modules generic Call() API * -------------------------------------------------------------------------- */ @@ -3162,9 +3709,8 @@ void moduleParseCallReply_Array(RedisModuleCallReply *reply) { reply->type = REDISMODULE_REPLY_ARRAY; } -/* Free a Call reply and all the nested replies it contains if it's an - * array. */ -void RM_FreeCallReply_Rec(RedisModuleCallReply *reply, int freenested){ +/* Recursive free reply function. */ +void moduleFreeCallReplyRec(RedisModuleCallReply *reply, int freenested){ /* Don't free nested replies by default: the user must always free the * toplevel reply. However be gentle and don't crash if the module * misuses the API. */ @@ -3174,7 +3720,7 @@ void RM_FreeCallReply_Rec(RedisModuleCallReply *reply, int freenested){ if (reply->type == REDISMODULE_REPLY_ARRAY) { size_t j; for (j = 0; j < reply->len; j++) - RM_FreeCallReply_Rec(reply->val.array+j,1); + moduleFreeCallReplyRec(reply->val.array+j,1); zfree(reply->val.array); } } @@ -3189,13 +3735,14 @@ void RM_FreeCallReply_Rec(RedisModuleCallReply *reply, int freenested){ } } -/* Wrapper for the recursive free reply function. This is needed in order - * to have the first level function to return on nested replies, but only - * if called by the module API. */ +/* Free a Call reply and all the nested replies it contains if it's an + * array. */ void RM_FreeCallReply(RedisModuleCallReply *reply) { - + /* This is a wrapper for the recursive free reply function. This is needed + * in order to have the first level function to return on nested replies, + * but only if called by the module API. */ RedisModuleCtx *ctx = reply->ctx; - RM_FreeCallReply_Rec(reply,0); + moduleFreeCallReplyRec(reply,0); autoMemoryFreed(ctx,REDISMODULE_AM_REPLY,reply); } @@ -3347,30 +3894,31 @@ fmterr: * * * **cmdname**: The Redis command to call. * * **fmt**: A format specifier string for the command's arguments. Each - * of the arguments should be specified by a valid type specification: - * b The argument is a buffer and is immediately followed by another - * argument that is the buffer's length. - * c The argument is a pointer to a plain C string (null-terminated). - * l The argument is long long integer. - * s The argument is a RedisModuleString. - * v The argument(s) is a vector of RedisModuleString. + * of the arguments should be specified by a valid type specification. The + * format specifier can also contain the modifiers `!`, `A` and `R` which + * don't have a corresponding argument. * - * The format specifier can also include modifiers: - * ! Sends the Redis command and its arguments to replicas and AOF. - * A Suppress AOF propagation, send only to replicas (requires `!`). - * R Suppress replicas propagation, send only to AOF (requires `!`). + * * `b` -- The argument is a buffer and is immediately followed by another + * argument that is the buffer's length. + * * `c` -- The argument is a pointer to a plain C string (null-terminated). + * * `l` -- The argument is long long integer. + * * `s` -- The argument is a RedisModuleString. + * * `v` -- The argument(s) is a vector of RedisModuleString. + * * `!` -- Sends the Redis command and its arguments to replicas and AOF. + * * `A` -- Suppress AOF propagation, send only to replicas (requires `!`). + * * `R` -- Suppress replicas propagation, send only to AOF (requires `!`). * * **...**: The actual arguments to the Redis command. * * On success a RedisModuleCallReply object is returned, otherwise * NULL is returned and errno is set to the following values: * - * EBADF: wrong format specifier. - * EINVAL: wrong command arity. - * ENOENT: command does not exist. - * EPERM: operation in Cluster instance with key in non local slot. - * EROFS: operation in Cluster instance when a write command is sent - * in a readonly state. - * ENETDOWN: operation in Cluster instance when cluster is down. + * * EBADF: wrong format specifier. + * * EINVAL: wrong command arity. + * * ENOENT: command does not exist. + * * EPERM: operation in Cluster instance with key in non local slot. + * * EROFS: operation in Cluster instance when a write command is sent + * in a readonly state. + * * ENETDOWN: operation in Cluster instance when cluster is down. * * Example code fragment: * @@ -3682,27 +4230,28 @@ robj *moduleTypeDupOrReply(client *c, robj *fromkey, robj *tokey, robj *value) { * still load old data produced by an older version if the rdb_load * callback is able to check the encver value and act accordingly. * The encver must be a positive value between 0 and 1023. + * * * **typemethods_ptr** is a pointer to a RedisModuleTypeMethods structure * that should be populated with the methods callbacks and structure * version, like in the following example: * - * RedisModuleTypeMethods tm = { - * .version = REDISMODULE_TYPE_METHOD_VERSION, - * .rdb_load = myType_RDBLoadCallBack, - * .rdb_save = myType_RDBSaveCallBack, - * .aof_rewrite = myType_AOFRewriteCallBack, - * .free = myType_FreeCallBack, + * RedisModuleTypeMethods tm = { + * .version = REDISMODULE_TYPE_METHOD_VERSION, + * .rdb_load = myType_RDBLoadCallBack, + * .rdb_save = myType_RDBSaveCallBack, + * .aof_rewrite = myType_AOFRewriteCallBack, + * .free = myType_FreeCallBack, * - * // Optional fields - * .digest = myType_DigestCallBack, - * .mem_usage = myType_MemUsageCallBack, - * .aux_load = myType_AuxRDBLoadCallBack, - * .aux_save = myType_AuxRDBSaveCallBack, - * .free_effort = myType_FreeEffortCallBack, - * .unlink = myType_UnlinkCallBack, - * .copy = myType_CopyCallback, - * .defrag = myType_DefragCallback - * } + * // Optional fields + * .digest = myType_DigestCallBack, + * .mem_usage = myType_MemUsageCallBack, + * .aux_load = myType_AuxRDBLoadCallBack, + * .aux_save = myType_AuxRDBSaveCallBack, + * .free_effort = myType_FreeEffortCallBack, + * .unlink = myType_UnlinkCallBack, + * .copy = myType_CopyCallback, + * .defrag = myType_DefragCallback + * } * * * **rdb_load**: A callback function pointer that loads data from RDB files. * * **rdb_save**: A callback function pointer that saves data to RDB files. @@ -3740,7 +4289,7 @@ robj *moduleTypeDupOrReply(client *c, robj *fromkey, robj *tokey, robj *value) { * a time limit and provides cursor support is used only for keys that are determined * to have significant internal complexity. To determine this, the defrag mechanism * uses the free_effort callback and the 'active-defrag-max-scan-fields' config directive. - * NOTE: The value is passed as a void** and the function is expected to update the + * NOTE: The value is passed as a `void**` and the function is expected to update the * pointer if the top-level value pointer is defragmented and consequentially changes. * * Note: the module name "AAAAAAAAA" is reserved and produces an error, it @@ -3900,7 +4449,7 @@ int moduleAllDatatypesHandleErrors() { } /* Returns true if any previous IO API failed. - * for Load* APIs the REDISMODULE_OPTIONS_HANDLE_IO_ERRORS flag must be set with + * for `Load*` APIs the REDISMODULE_OPTIONS_HANDLE_IO_ERRORS flag must be set with * RedisModule_SetModuleOptions first. */ int RM_IsIOError(RedisModuleIO *io) { return io->error; @@ -3926,7 +4475,7 @@ saveerr: } /* Load an unsigned 64 bit value from the RDB file. This function should only - * be called in the context of the rdb_load method of modules implementing + * be called in the context of the `rdb_load` method of modules implementing * new data types. */ uint64_t RM_LoadUnsigned(RedisModuleIO *io) { if (io->error) return 0; @@ -4242,7 +4791,6 @@ void RM_DigestEndSequence(RedisModuleDigest *md) { * If this is NOT done, Redis will handle corrupted (or just truncated) serialized * data by producing an error message and terminating the process. */ - void *RM_LoadDataTypeFromString(const RedisModuleString *str, const moduleType *mt) { rio payload; RedisModuleIO io; @@ -4270,7 +4818,6 @@ void *RM_LoadDataTypeFromString(const RedisModuleString *str, const moduleType * * implement in order to allow a module to arbitrarily serialize/de-serialize * keys, similar to how the Redis 'DUMP' and 'RESTORE' commands are implemented. */ - RedisModuleString *RM_SaveDataTypeToString(RedisModuleCtx *ctx, void *data, const moduleType *mt) { rio payload; RedisModuleIO io; @@ -4368,7 +4915,7 @@ const RedisModuleString *RM_GetKeyNameFromIO(RedisModuleIO *io) { return io->key; } -/* Returns a RedisModuleString with the name of the key from RedisModuleKey */ +/* Returns a RedisModuleString with the name of the key from RedisModuleKey. */ const RedisModuleString *RM_GetKeyNameFromModuleKey(RedisModuleKey *key) { return key ? key->key : NULL; } @@ -4383,7 +4930,7 @@ const RedisModuleString *RM_GetKeyNameFromModuleKey(RedisModuleKey *key) { * RM_LogIOError() * */ -void RM_LogRaw(RedisModule *module, const char *levelstr, const char *fmt, va_list ap) { +void moduleLogRaw(RedisModule *module, const char *levelstr, const char *fmt, va_list ap) { char msg[LOG_MAX_LEN]; size_t name_len; int level; @@ -4422,7 +4969,7 @@ void RM_LogRaw(RedisModule *module, const char *levelstr, const char *fmt, va_li void RM_Log(RedisModuleCtx *ctx, const char *levelstr, const char *fmt, ...) { va_list ap; va_start(ap, fmt); - RM_LogRaw(ctx? ctx->module: NULL,levelstr,fmt,ap); + moduleLogRaw(ctx? ctx->module: NULL,levelstr,fmt,ap); va_end(ap); } @@ -4434,11 +4981,14 @@ void RM_Log(RedisModuleCtx *ctx, const char *levelstr, const char *fmt, ...) { void RM_LogIOError(RedisModuleIO *io, const char *levelstr, const char *fmt, ...) { va_list ap; va_start(ap, fmt); - RM_LogRaw(io->type->module,levelstr,fmt,ap); + moduleLogRaw(io->type->module,levelstr,fmt,ap); va_end(ap); } /* Redis-like assert function. + * + * The macro `RedisModule_Assert(expression)` is recommended, rather than + * calling this function directly. * * A failed assertion will shut down the server and produce logging information * that looks identical to information generated by Redis itself. @@ -4570,6 +5120,7 @@ RedisModuleBlockedClient *moduleBlockClient(RedisModuleCtx *ctx, RedisModuleCmdF bc->dbid = c->db->id; bc->blocked_on_keys = keys != NULL; bc->unblocked = 0; + bc->background_duration = 0; c->bpop.timeout = timeout; if (islua || ismulti) { @@ -4643,6 +5194,11 @@ int moduleTryServeClientBlockedOnKey(client *c, robj *key) { * * In these cases, a call to RedisModule_BlockClient() will **not** block the * client, but instead produce a specific error reply. + * + * Measuring background time: By default the time spent in the blocked command + * is not account for the total command duration. To include such time you should + * use RM_BlockedClientMeasureTimeStart() and RM_BlockedClientMeasureTimeEnd() one, + * or multiple times within the blocking command background work. */ RedisModuleBlockedClient *RM_BlockClient(RedisModuleCtx *ctx, RedisModuleCmdFunc reply_callback, RedisModuleCmdFunc timeout_callback, void (*free_privdata)(RedisModuleCtx*,void*), long long timeout_ms) { return moduleBlockClient(ctx,reply_callback,timeout_callback,free_privdata,timeout_ms, NULL,0,NULL); @@ -4673,7 +5229,7 @@ RedisModuleBlockedClient *RM_BlockClient(RedisModuleCtx *ctx, RedisModuleCmdFunc * key, or a client in queue before this one can be served, modifying the key * as well and making it empty again. So when a client is blocked with * RedisModule_BlockClientOnKeys() the reply callback is not called after - * RM_UnblockCLient() is called, but every time a key is signaled as ready: + * RM_UnblockClient() is called, but every time a key is signaled as ready: * if the reply callback can serve the client, it returns REDISMODULE_OK * and the client is unblocked, otherwise it will return REDISMODULE_ERR * and we'll try again later. @@ -4837,6 +5393,7 @@ void moduleHandleBlockedClients(void) { * was blocked on keys (RM_BlockClientOnKeys()), because we already * called such callback in moduleTryServeClientBlockedOnKey() when * the key was signaled as ready. */ + uint64_t reply_us = 0; if (c && !bc->blocked_on_keys && bc->reply_callback) { RedisModuleCtx ctx = REDISMODULE_CTX_INIT; ctx.flags |= REDISMODULE_CTX_BLOCKED_REPLY; @@ -4845,9 +5402,19 @@ void moduleHandleBlockedClients(void) { ctx.module = bc->module; ctx.client = bc->client; ctx.blocked_client = bc; + monotime replyTimer; + elapsedStart(&replyTimer); bc->reply_callback(&ctx,(void**)c->argv,c->argc); + reply_us = elapsedUs(replyTimer); moduleFreeContext(&ctx); } + /* Update stats now that we've finished the blocking operation. + * This needs to be out of the reply callback above given that a + * module might not define any callback and still do blocking ops. + */ + if (c && !bc->blocked_on_keys) { + updateStatsOnUnblock(c, bc->background_duration, reply_us); + } /* Free privdata if any. */ if (bc->privdata && bc->free_privdata) { @@ -4911,6 +5478,9 @@ void moduleBlockedClientTimedOut(client *c) { ctx.blocked_privdata = bc->privdata; bc->timeout_callback(&ctx,(void**)c->argv,c->argc); moduleFreeContext(&ctx); + if (!bc->blocked_on_keys) { + updateStatsOnUnblock(c, bc->background_duration, 0); + } /* For timeout events, we do not want to call the disconnect callback, * because the blocked client will be automatically disconnected in * this case, and the user can still hook using the timeout callback. */ @@ -5103,9 +5673,9 @@ void moduleReleaseGIL(void) { * * The subscriber signature is: * - * int (*RedisModuleNotificationFunc) (RedisModuleCtx *ctx, int type, - * const char *event, - * RedisModuleString *key); + * int (*RedisModuleNotificationFunc) (RedisModuleCtx *ctx, int type, + * const char *event, + * RedisModuleString *key); * * `type` is the event type bit, that must match the mask given at registration * time. The event string is the actual command being executed, and key is the @@ -5369,28 +5939,27 @@ size_t RM_GetClusterSize(void) { return dictSize(server.cluster->nodes); } +clusterNode *clusterLookupNode(const char *name); /* We need access to internals */ + /* Populate the specified info for the node having as ID the specified 'id', * then returns REDISMODULE_OK. Otherwise if the node ID does not exist from * the POV of this local node, REDISMODULE_ERR is returned. * - * The arguments ip, master_id, port and flags can be NULL in case we don't - * need to populate back certain info. If an ip and master_id (only populated + * The arguments `ip`, `master_id`, `port` and `flags` can be NULL in case we don't + * need to populate back certain info. If an `ip` and `master_id` (only populated * if the instance is a slave) are specified, they point to buffers holding - * at least REDISMODULE_NODE_ID_LEN bytes. The strings written back as ip - * and master_id are not null terminated. + * at least REDISMODULE_NODE_ID_LEN bytes. The strings written back as `ip` + * and `master_id` are not null terminated. * * The list of flags reported is the following: * - * * REDISMODULE_NODE_MYSELF This node - * * REDISMODULE_NODE_MASTER The node is a master - * * REDISMODULE_NODE_SLAVE The node is a replica - * * REDISMODULE_NODE_PFAIL We see the node as failing - * * REDISMODULE_NODE_FAIL The cluster agrees the node is failing - * * REDISMODULE_NODE_NOFAILOVER The slave is configured to never failover + * * REDISMODULE_NODE_MYSELF: This node + * * REDISMODULE_NODE_MASTER: The node is a master + * * REDISMODULE_NODE_SLAVE: The node is a replica + * * REDISMODULE_NODE_PFAIL: We see the node as failing + * * REDISMODULE_NODE_FAIL: The cluster agrees the node is failing + * * REDISMODULE_NODE_NOFAILOVER: The slave is configured to never failover */ - -clusterNode *clusterLookupNode(const char *name); /* We need access to internals */ - int RM_GetClusterNodeInfo(RedisModuleCtx *ctx, const char *id, char *ip, char *master_id, int *port, int *flags) { UNUSED(ctx); @@ -5434,18 +6003,18 @@ int RM_GetClusterNodeInfo(RedisModuleCtx *ctx, const char *id, char *ip, char *m * a different distributed system, but still want to use the Redis Cluster * message bus. Flags that can be set: * - * CLUSTER_MODULE_FLAG_NO_FAILOVER - * CLUSTER_MODULE_FLAG_NO_REDIRECTION + * * CLUSTER_MODULE_FLAG_NO_FAILOVER + * * CLUSTER_MODULE_FLAG_NO_REDIRECTION * * With the following effects: * - * NO_FAILOVER: prevent Redis Cluster slaves to failover a failing master. - * Also disables the replica migration feature. + * * NO_FAILOVER: prevent Redis Cluster slaves to failover a failing master. + * Also disables the replica migration feature. * - * NO_REDIRECTION: Every node will accept any key, without trying to perform - * partitioning according to the user Redis Cluster algorithm. - * Slots informations will still be propagated across the - * cluster, but without effects. */ + * * NO_REDIRECTION: Every node will accept any key, without trying to perform + * partitioning according to the user Redis Cluster algorithm. + * Slots informations will still be propagated across the + * cluster, but without effects. */ void RM_SetClusterFlags(RedisModuleCtx *ctx, uint64_t flags) { UNUSED(ctx); if (flags & REDISMODULE_CLUSTER_FLAG_NO_FAILOVER) @@ -5964,15 +6533,15 @@ int RM_DictDel(RedisModuleDict *d, RedisModuleString *key, void *oldval) { * comparison operator to use in order to seek the first element. The * operators available are: * - * "^" -- Seek the first (lexicographically smaller) key. - * "$" -- Seek the last (lexicographically biffer) key. - * ">" -- Seek the first element greater than the specified key. - * ">=" -- Seek the first element greater or equal than the specified key. - * "<" -- Seek the first element smaller than the specified key. - * "<=" -- Seek the first element smaller or equal than the specified key. - * "==" -- Seek the first element matching exactly the specified key. + * * `^` -- Seek the first (lexicographically smaller) key. + * * `$` -- Seek the last (lexicographically biffer) key. + * * `>` -- Seek the first element greater than the specified key. + * * `>=` -- Seek the first element greater or equal than the specified key. + * * `<` -- Seek the first element smaller than the specified key. + * * `<=` -- Seek the first element smaller or equal than the specified key. + * * `==` -- Seek the first element matching exactly the specified key. * - * Note that for "^" and "$" the passed key is not used, and the user may + * Note that for `^` and `$` the passed key is not used, and the user may * just pass NULL with a length of 0. * * If the element to start the iteration cannot be seeked based on the @@ -6017,11 +6586,11 @@ int RM_DictIteratorReseek(RedisModuleDictIter *di, const char *op, RedisModuleSt return RM_DictIteratorReseekC(di,op,key->ptr,sdslen(key->ptr)); } -/* Return the current item of the dictionary iterator 'di' and steps to the +/* Return the current item of the dictionary iterator `di` and steps to the * next element. If the iterator already yield the last element and there * are no other elements to return, NULL is returned, otherwise a pointer - * to a string representing the key is provided, and the '*keylen' length - * is set by reference (if keylen is not NULL). The '*dataptr', if not NULL + * to a string representing the key is provided, and the `*keylen` length + * is set by reference (if keylen is not NULL). The `*dataptr`, if not NULL * is set to the value of the pointer stored at the returned key as auxiliary * data (as set by the RedisModule_DictSet API). * @@ -6035,7 +6604,7 @@ int RM_DictIteratorReseek(RedisModuleDictIter *di, const char *op, RedisModuleSt * } * * The returned pointer is of type void because sometimes it makes sense - * to cast it to a char* sometimes to an unsigned char* depending on the + * to cast it to a `char*` sometimes to an unsigned `char*` depending on the * fact it contains or not binary data, so this API ends being more * comfortable to use. * @@ -6119,8 +6688,8 @@ int RM_DictCompare(RedisModuleDictIter *di, const char *op, RedisModuleString *k int RM_InfoEndDictField(RedisModuleInfoCtx *ctx); /* Used to start a new section, before adding any fields. the section name will - * be prefixed by "_" and must only include A-Z,a-z,0-9. - * NULL or empty string indicates the default section (only ) is used. + * be prefixed by `_` and must only include A-Z,a-z,0-9. + * NULL or empty string indicates the default section (only ``) is used. * When return value is REDISMODULE_ERR, the section should and will be skipped. */ int RM_InfoAddSection(RedisModuleInfoCtx *ctx, char *name) { sds full_name = sdsdup(ctx->module->name); @@ -6180,8 +6749,8 @@ int RM_InfoEndDictField(RedisModuleInfoCtx *ctx) { } /* Used by RedisModuleInfoFunc to add info fields. - * Each field will be automatically prefixed by "_". - * Field names or values must not include \r\n of ":" */ + * Each field will be automatically prefixed by `_`. + * Field names or values must not include `\r\n` or `:`. */ int RM_InfoAddFieldString(RedisModuleInfoCtx *ctx, char *field, RedisModuleString *value) { if (!ctx->in_section) return REDISMODULE_ERR; @@ -6200,6 +6769,7 @@ int RM_InfoAddFieldString(RedisModuleInfoCtx *ctx, char *field, RedisModuleStrin return REDISMODULE_OK; } +/* See RedisModule_InfoAddFieldString(). */ int RM_InfoAddFieldCString(RedisModuleInfoCtx *ctx, char *field, char *value) { if (!ctx->in_section) return REDISMODULE_ERR; @@ -6218,6 +6788,7 @@ int RM_InfoAddFieldCString(RedisModuleInfoCtx *ctx, char *field, char *value) { return REDISMODULE_OK; } +/* See RedisModule_InfoAddFieldString(). */ int RM_InfoAddFieldDouble(RedisModuleInfoCtx *ctx, char *field, double value) { if (!ctx->in_section) return REDISMODULE_ERR; @@ -6236,6 +6807,7 @@ int RM_InfoAddFieldDouble(RedisModuleInfoCtx *ctx, char *field, double value) { return REDISMODULE_OK; } +/* See RedisModule_InfoAddFieldString(). */ int RM_InfoAddFieldLongLong(RedisModuleInfoCtx *ctx, char *field, long long value) { if (!ctx->in_section) return REDISMODULE_ERR; @@ -6254,6 +6826,7 @@ int RM_InfoAddFieldLongLong(RedisModuleInfoCtx *ctx, char *field, long long valu return REDISMODULE_OK; } +/* See RedisModule_InfoAddFieldString(). */ int RM_InfoAddFieldULongLong(RedisModuleInfoCtx *ctx, char *field, unsigned long long value) { if (!ctx->in_section) return REDISMODULE_ERR; @@ -6272,6 +6845,8 @@ int RM_InfoAddFieldULongLong(RedisModuleInfoCtx *ctx, char *field, unsigned long return REDISMODULE_OK; } +/* Registers callback for the INFO command. The callback should add INFO fields + * by calling the `RedisModule_InfoAddField*()` functions. */ int RM_RegisterInfoFunc(RedisModuleCtx *ctx, RedisModuleInfoFunc cb) { ctx->module->info_cb = cb; return REDISMODULE_OK; @@ -6711,7 +7286,6 @@ const RedisModuleString *RM_CommandFilterArgGet(RedisModuleCommandFilterCtx *fct * after the filter context is destroyed, so it must not be auto-memory * allocated, freed or used elsewhere. */ - int RM_CommandFilterArgInsert(RedisModuleCommandFilterCtx *fctx, int pos, RedisModuleString *arg) { int i; @@ -6733,7 +7307,6 @@ int RM_CommandFilterArgInsert(RedisModuleCommandFilterCtx *fctx, int pos, RedisM * filter context is destroyed, so it must not be auto-memory allocated, freed * or used elsewhere. */ - int RM_CommandFilterArgReplace(RedisModuleCommandFilterCtx *fctx, int pos, RedisModuleString *arg) { if (pos < 0 || pos >= fctx->argc) return REDISMODULE_ERR; @@ -6774,10 +7347,10 @@ size_t RM_MallocSize(void* ptr){ /* Return the a number between 0 to 1 indicating the amount of memory * currently used, relative to the Redis "maxmemory" configuration. * - * 0 - No memory limit configured. - * Between 0 and 1 - The percentage of the memory used normalized in 0-1 range. - * Exactly 1 - Memory limit reached. - * Greater 1 - More memory used than the configured limit. + * * 0 - No memory limit configured. + * * Between 0 and 1 - The percentage of the memory used normalized in 0-1 range. + * * Exactly 1 - Memory limit reached. + * * Greater 1 - More memory used than the configured limit. */ float RM_GetUsedMemoryRatio(){ float level; @@ -6840,21 +7413,22 @@ void RM_ScanCursorDestroy(RedisModuleScanCursor *cursor) { * the selected db. * * Callback for scan implementation. - * void scan_callback(RedisModuleCtx *ctx, RedisModuleString *keyname, - * RedisModuleKey *key, void *privdata); - * ctx - the redis module context provided to for the scan. - * keyname - owned by the caller and need to be retained if used after this - * function. * - * key - holds info on the key and value, it is provided as best effort, in - * some cases it might be NULL, in which case the user should (can) use - * RedisModule_OpenKey (and CloseKey too). - * when it is provided, it is owned by the caller and will be free when the - * callback returns. + * void scan_callback(RedisModuleCtx *ctx, RedisModuleString *keyname, + * RedisModuleKey *key, void *privdata); * - * privdata - the user data provided to RedisModule_Scan. + * - `ctx`: the redis module context provided to for the scan. + * - `keyname`: owned by the caller and need to be retained if used after this + * function. + * - `key`: holds info on the key and value, it is provided as best effort, in + * some cases it might be NULL, in which case the user should (can) use + * RedisModule_OpenKey() (and CloseKey too). + * when it is provided, it is owned by the caller and will be free when the + * callback returns. + * - `privdata`: the user data provided to RedisModule_Scan(). * * The way it should be used: + * * RedisModuleCursor *c = RedisModule_ScanCursorCreate(); * while(RedisModule_Scan(ctx, c, callback, privateData)); * RedisModule_ScanCursorDestroy(c); @@ -6938,7 +7512,9 @@ static void moduleScanKeyCallback(void *privdata, const dictEntry *de) { /* Scan api that allows a module to scan the elements in a hash, set or sorted set key * * Callback for scan implementation. - * void scan_callback(RedisModuleKey *key, RedisModuleString* field, RedisModuleString* value, void *privdata); + * + * void scan_callback(RedisModuleKey *key, RedisModuleString* field, RedisModuleString* value, void *privdata); + * * - key - the redis key context provided to for the scan. * - field - field name, owned by the caller and need to be retained if used * after this function. @@ -6947,6 +7523,7 @@ static void moduleScanKeyCallback(void *privdata, const dictEntry *de) { * - privdata - the user data provided to RedisModule_ScanKey. * * The way it should be used: + * * RedisModuleCursor *c = RedisModule_ScanCursorCreate(); * RedisModuleKey *key = RedisModule_OpenKey(...) * while(RedisModule_ScanKey(key, c, callback, privateData)); @@ -6955,6 +7532,7 @@ static void moduleScanKeyCallback(void *privdata, const dictEntry *de) { * * It is also possible to use this API from another thread while the lock is acquired during * the actuall call to RM_ScanKey, and re-opening the key each time: + * * RedisModuleCursor *c = RedisModule_ScanCursorCreate(); * RedisModule_ThreadSafeContextLock(ctx); * RedisModuleKey *key = RedisModule_OpenKey(...) @@ -7159,10 +7737,10 @@ void ModuleForkDoneHandler(int exitcode, int bysignal) { * * The callback must be of this type: * - * int (*RedisModuleEventCallback)(RedisModuleCtx *ctx, - * RedisModuleEvent eid, - * uint64_t subevent, - * void *data); + * int (*RedisModuleEventCallback)(RedisModuleCtx *ctx, + * RedisModuleEvent eid, + * uint64_t subevent, + * void *data); * * The 'ctx' is a normal Redis module context that the callback can use in * order to call other modules APIs. The 'eid' is the event itself, this @@ -7176,201 +7754,207 @@ void ModuleForkDoneHandler(int exitcode, int bysignal) { * * Here is a list of events you can use as 'eid' and related sub events: * - * RedisModuleEvent_ReplicationRoleChanged + * * RedisModuleEvent_ReplicationRoleChanged: * - * This event is called when the instance switches from master - * to replica or the other way around, however the event is - * also called when the replica remains a replica but starts to - * replicate with a different master. + * This event is called when the instance switches from master + * to replica or the other way around, however the event is + * also called when the replica remains a replica but starts to + * replicate with a different master. * - * The following sub events are available: + * The following sub events are available: * - * REDISMODULE_SUBEVENT_REPLROLECHANGED_NOW_MASTER - * REDISMODULE_SUBEVENT_REPLROLECHANGED_NOW_REPLICA + * * `REDISMODULE_SUBEVENT_REPLROLECHANGED_NOW_MASTER` + * * `REDISMODULE_SUBEVENT_REPLROLECHANGED_NOW_REPLICA` * - * The 'data' field can be casted by the callback to a - * RedisModuleReplicationInfo structure with the following fields: + * The 'data' field can be casted by the callback to a + * `RedisModuleReplicationInfo` structure with the following fields: * - * int master; // true if master, false if replica - * char *masterhost; // master instance hostname for NOW_REPLICA - * int masterport; // master instance port for NOW_REPLICA - * char *replid1; // Main replication ID - * char *replid2; // Secondary replication ID - * uint64_t repl1_offset; // Main replication offset - * uint64_t repl2_offset; // Offset of replid2 validity + * int master; // true if master, false if replica + * char *masterhost; // master instance hostname for NOW_REPLICA + * int masterport; // master instance port for NOW_REPLICA + * char *replid1; // Main replication ID + * char *replid2; // Secondary replication ID + * uint64_t repl1_offset; // Main replication offset + * uint64_t repl2_offset; // Offset of replid2 validity * - * RedisModuleEvent_Persistence + * * RedisModuleEvent_Persistence * - * This event is called when RDB saving or AOF rewriting starts - * and ends. The following sub events are available: + * This event is called when RDB saving or AOF rewriting starts + * and ends. The following sub events are available: * - * REDISMODULE_SUBEVENT_PERSISTENCE_RDB_START - * REDISMODULE_SUBEVENT_PERSISTENCE_AOF_START - * REDISMODULE_SUBEVENT_PERSISTENCE_SYNC_RDB_START - * REDISMODULE_SUBEVENT_PERSISTENCE_ENDED - * REDISMODULE_SUBEVENT_PERSISTENCE_FAILED + * * `REDISMODULE_SUBEVENT_PERSISTENCE_RDB_START` + * * `REDISMODULE_SUBEVENT_PERSISTENCE_AOF_START` + * * `REDISMODULE_SUBEVENT_PERSISTENCE_SYNC_RDB_START` + * * `REDISMODULE_SUBEVENT_PERSISTENCE_ENDED` + * * `REDISMODULE_SUBEVENT_PERSISTENCE_FAILED` * - * The above events are triggered not just when the user calls the - * relevant commands like BGSAVE, but also when a saving operation - * or AOF rewriting occurs because of internal server triggers. - * The SYNC_RDB_START sub events are happening in the forground due to - * SAVE command, FLUSHALL, or server shutdown, and the other RDB and - * AOF sub events are executed in a background fork child, so any - * action the module takes can only affect the generated AOF or RDB, - * but will not be reflected in the parent process and affect connected - * clients and commands. Also note that the AOF_START sub event may end - * up saving RDB content in case of an AOF with rdb-preamble. + * The above events are triggered not just when the user calls the + * relevant commands like BGSAVE, but also when a saving operation + * or AOF rewriting occurs because of internal server triggers. + * The SYNC_RDB_START sub events are happening in the forground due to + * SAVE command, FLUSHALL, or server shutdown, and the other RDB and + * AOF sub events are executed in a background fork child, so any + * action the module takes can only affect the generated AOF or RDB, + * but will not be reflected in the parent process and affect connected + * clients and commands. Also note that the AOF_START sub event may end + * up saving RDB content in case of an AOF with rdb-preamble. * - * RedisModuleEvent_FlushDB + * * RedisModuleEvent_FlushDB * - * The FLUSHALL, FLUSHDB or an internal flush (for instance - * because of replication, after the replica synchronization) - * happened. The following sub events are available: + * The FLUSHALL, FLUSHDB or an internal flush (for instance + * because of replication, after the replica synchronization) + * happened. The following sub events are available: * - * REDISMODULE_SUBEVENT_FLUSHDB_START - * REDISMODULE_SUBEVENT_FLUSHDB_END + * * `REDISMODULE_SUBEVENT_FLUSHDB_START` + * * `REDISMODULE_SUBEVENT_FLUSHDB_END` * - * The data pointer can be casted to a RedisModuleFlushInfo - * structure with the following fields: + * The data pointer can be casted to a RedisModuleFlushInfo + * structure with the following fields: * - * int32_t async; // True if the flush is done in a thread. - * See for instance FLUSHALL ASYNC. - * In this case the END callback is invoked - * immediately after the database is put - * in the free list of the thread. - * int32_t dbnum; // Flushed database number, -1 for all the DBs - * in the case of the FLUSHALL operation. + * int32_t async; // True if the flush is done in a thread. + * // See for instance FLUSHALL ASYNC. + * // In this case the END callback is invoked + * // immediately after the database is put + * // in the free list of the thread. + * int32_t dbnum; // Flushed database number, -1 for all the DBs + * // in the case of the FLUSHALL operation. * - * The start event is called *before* the operation is initated, thus - * allowing the callback to call DBSIZE or other operation on the - * yet-to-free keyspace. + * The start event is called *before* the operation is initated, thus + * allowing the callback to call DBSIZE or other operation on the + * yet-to-free keyspace. * - * RedisModuleEvent_Loading + * * RedisModuleEvent_Loading * - * Called on loading operations: at startup when the server is - * started, but also after a first synchronization when the - * replica is loading the RDB file from the master. - * The following sub events are available: + * Called on loading operations: at startup when the server is + * started, but also after a first synchronization when the + * replica is loading the RDB file from the master. + * The following sub events are available: * - * REDISMODULE_SUBEVENT_LOADING_RDB_START - * REDISMODULE_SUBEVENT_LOADING_AOF_START - * REDISMODULE_SUBEVENT_LOADING_REPL_START - * REDISMODULE_SUBEVENT_LOADING_ENDED - * REDISMODULE_SUBEVENT_LOADING_FAILED + * * `REDISMODULE_SUBEVENT_LOADING_RDB_START` + * * `REDISMODULE_SUBEVENT_LOADING_AOF_START` + * * `REDISMODULE_SUBEVENT_LOADING_REPL_START` + * * `REDISMODULE_SUBEVENT_LOADING_ENDED` + * * `REDISMODULE_SUBEVENT_LOADING_FAILED` * - * Note that AOF loading may start with an RDB data in case of - * rdb-preamble, in which case you'll only receive an AOF_START event. + * Note that AOF loading may start with an RDB data in case of + * rdb-preamble, in which case you'll only receive an AOF_START event. * + * * RedisModuleEvent_ClientChange * - * RedisModuleEvent_ClientChange + * Called when a client connects or disconnects. + * The data pointer can be casted to a RedisModuleClientInfo + * structure, documented in RedisModule_GetClientInfoById(). + * The following sub events are available: * - * Called when a client connects or disconnects. - * The data pointer can be casted to a RedisModuleClientInfo - * structure, documented in RedisModule_GetClientInfoById(). - * The following sub events are available: + * * `REDISMODULE_SUBEVENT_CLIENT_CHANGE_CONNECTED` + * * `REDISMODULE_SUBEVENT_CLIENT_CHANGE_DISCONNECTED` * - * REDISMODULE_SUBEVENT_CLIENT_CHANGE_CONNECTED - * REDISMODULE_SUBEVENT_CLIENT_CHANGE_DISCONNECTED + * * RedisModuleEvent_Shutdown * - * RedisModuleEvent_Shutdown + * The server is shutting down. No subevents are available. * - * The server is shutting down. No subevents are available. + * * RedisModuleEvent_ReplicaChange * - * RedisModuleEvent_ReplicaChange + * This event is called when the instance (that can be both a + * master or a replica) get a new online replica, or lose a + * replica since it gets disconnected. + * The following sub events are available: * - * This event is called when the instance (that can be both a - * master or a replica) get a new online replica, or lose a - * replica since it gets disconnected. - * The following sub events are available: + * * `REDISMODULE_SUBEVENT_REPLICA_CHANGE_ONLINE` + * * `REDISMODULE_SUBEVENT_REPLICA_CHANGE_OFFLINE` * - * REDISMODULE_SUBEVENT_REPLICA_CHANGE_ONLINE - * REDISMODULE_SUBEVENT_REPLICA_CHANGE_OFFLINE + * No additional information is available so far: future versions + * of Redis will have an API in order to enumerate the replicas + * connected and their state. * - * No additional information is available so far: future versions - * of Redis will have an API in order to enumerate the replicas - * connected and their state. + * * RedisModuleEvent_CronLoop * - * RedisModuleEvent_CronLoop + * This event is called every time Redis calls the serverCron() + * function in order to do certain bookkeeping. Modules that are + * required to do operations from time to time may use this callback. + * Normally Redis calls this function 10 times per second, but + * this changes depending on the "hz" configuration. + * No sub events are available. * - * This event is called every time Redis calls the serverCron() - * function in order to do certain bookkeeping. Modules that are - * required to do operations from time to time may use this callback. - * Normally Redis calls this function 10 times per second, but - * this changes depending on the "hz" configuration. - * No sub events are available. + * The data pointer can be casted to a RedisModuleCronLoop + * structure with the following fields: * - * The data pointer can be casted to a RedisModuleCronLoop - * structure with the following fields: + * int32_t hz; // Approximate number of events per second. * - * int32_t hz; // Approximate number of events per second. + * * RedisModuleEvent_MasterLinkChange * - * RedisModuleEvent_MasterLinkChange + * This is called for replicas in order to notify when the + * replication link becomes functional (up) with our master, + * or when it goes down. Note that the link is not considered + * up when we just connected to the master, but only if the + * replication is happening correctly. + * The following sub events are available: * - * This is called for replicas in order to notify when the - * replication link becomes functional (up) with our master, - * or when it goes down. Note that the link is not considered - * up when we just connected to the master, but only if the - * replication is happening correctly. - * The following sub events are available: + * * `REDISMODULE_SUBEVENT_MASTER_LINK_UP` + * * `REDISMODULE_SUBEVENT_MASTER_LINK_DOWN` * - * REDISMODULE_SUBEVENT_MASTER_LINK_UP - * REDISMODULE_SUBEVENT_MASTER_LINK_DOWN + * * RedisModuleEvent_ModuleChange * - * RedisModuleEvent_ModuleChange + * This event is called when a new module is loaded or one is unloaded. + * The following sub events are available: * - * This event is called when a new module is loaded or one is unloaded. - * The following sub events are available: + * * `REDISMODULE_SUBEVENT_MODULE_LOADED` + * * `REDISMODULE_SUBEVENT_MODULE_UNLOADED` * - * REDISMODULE_SUBEVENT_MODULE_LOADED - * REDISMODULE_SUBEVENT_MODULE_UNLOADED + * The data pointer can be casted to a RedisModuleModuleChange + * structure with the following fields: * - * The data pointer can be casted to a RedisModuleModuleChange - * structure with the following fields: + * const char* module_name; // Name of module loaded or unloaded. + * int32_t module_version; // Module version. * - * const char* module_name; // Name of module loaded or unloaded. - * int32_t module_version; // Module version. + * * RedisModuleEvent_LoadingProgress * - * RedisModuleEvent_LoadingProgress + * This event is called repeatedly called while an RDB or AOF file + * is being loaded. + * The following sub events are availble: * - * This event is called repeatedly called while an RDB or AOF file - * is being loaded. - * The following sub events are availble: + * * `REDISMODULE_SUBEVENT_LOADING_PROGRESS_RDB` + * * `REDISMODULE_SUBEVENT_LOADING_PROGRESS_AOF` * - * REDISMODULE_SUBEVENT_LOADING_PROGRESS_RDB - * REDISMODULE_SUBEVENT_LOADING_PROGRESS_AOF + * The data pointer can be casted to a RedisModuleLoadingProgress + * structure with the following fields: * - * The data pointer can be casted to a RedisModuleLoadingProgress - * structure with the following fields: + * int32_t hz; // Approximate number of events per second. + * int32_t progress; // Approximate progress between 0 and 1024, + * // or -1 if unknown. * - * int32_t hz; // Approximate number of events per second. - * int32_t progress; // Approximate progress between 0 and 1024, - * or -1 if unknown. + * * RedisModuleEvent_SwapDB * - * RedisModuleEvent_SwapDB + * This event is called when a SWAPDB command has been successfully + * Executed. + * For this event call currently there is no subevents available. * - * This event is called when a SWAPDB command has been successfully - * Executed. - * For this event call currently there is no subevents available. + * The data pointer can be casted to a RedisModuleSwapDbInfo + * structure with the following fields: * - * The data pointer can be casted to a RedisModuleSwapDbInfo - * structure with the following fields: + * int32_t dbnum_first; // Swap Db first dbnum + * int32_t dbnum_second; // Swap Db second dbnum * - * int32_t dbnum_first; // Swap Db first dbnum - * int32_t dbnum_second; // Swap Db second dbnum + * * RedisModuleEvent_ReplBackup * - * RedisModuleEvent_ReplBackup + * Called when diskless-repl-load config is set to swapdb, + * And redis needs to backup the the current database for the + * possibility to be restored later. A module with global data and + * maybe with aux_load and aux_save callbacks may need to use this + * notification to backup / restore / discard its globals. + * The following sub events are available: * - * Called when diskless-repl-load config is set to swapdb, - * And redis needs to backup the the current database for the - * possibility to be restored later. A module with global data and - * maybe with aux_load and aux_save callbacks may need to use this - * notification to backup / restore / discard its globals. - * The following sub events are available: + * * `REDISMODULE_SUBEVENT_REPL_BACKUP_CREATE` + * * `REDISMODULE_SUBEVENT_REPL_BACKUP_RESTORE` + * * `REDISMODULE_SUBEVENT_REPL_BACKUP_DISCARD` * - * REDISMODULE_SUBEVENT_REPL_BACKUP_CREATE - * REDISMODULE_SUBEVENT_REPL_BACKUP_RESTORE - * REDISMODULE_SUBEVENT_REPL_BACKUP_DISCARD + * * RedisModuleEvent_ForkChild * + * Called when a fork child (AOFRW, RDBSAVE, module fork...) is born/dies + * The following sub events are available: + * + * * `REDISMODULE_SUBEVENT_FORK_CHILD_BORN` + * * `REDISMODULE_SUBEVENT_FORK_CHILD_DIED` * * The function returns REDISMODULE_OK if the module was successfully subscribed * for the specified event. If the API is called from a wrong context or unsupported event @@ -7444,6 +8028,8 @@ int RM_IsSubEventSupported(RedisModuleEvent event, int64_t subevent) { return subevent < _REDISMODULE_SUBEVENT_SWAPDB_NEXT; case REDISMODULE_EVENT_REPL_BACKUP: return subevent < _REDISMODULE_SUBEVENT_REPL_BACKUP_NEXT; + case REDISMODULE_EVENT_FORK_CHILD: + return subevent < _REDISMODULE_SUBEVENT_FORK_CHILD_NEXT; default: break; } @@ -7659,6 +8245,11 @@ void moduleInitModulesSystem(void) { anetNonBlock(NULL,server.module_blocked_pipe[0]); anetNonBlock(NULL,server.module_blocked_pipe[1]); + /* Enable close-on-exec flag on pipes in case of the fork-exec system calls in + * sentinels or redis servers. */ + anetCloexec(server.module_blocked_pipe[0]); + anetCloexec(server.module_blocked_pipe[1]); + /* Create the timers radix tree. */ Timers = raxNew(); @@ -8064,7 +8655,8 @@ int RM_GetLFU(RedisModuleKey *key, long long *lfu_freq) { * the module can check if a certain set of flags are supported * by the redis server version in use. * Example: - * int supportedFlags = RM_GetContextFlagsAll() + * + * int supportedFlags = RM_GetContextFlagsAll(); * if (supportedFlags & REDISMODULE_CTX_FLAGS_MULTI) { * // REDISMODULE_CTX_FLAGS_MULTI is supported * } else{ @@ -8080,7 +8672,8 @@ int RM_GetContextFlagsAll() { * the module can check if a certain set of flags are supported * by the redis server version in use. * Example: - * int supportedFlags = RM_GetKeyspaceNotificationFlagsAll() + * + * int supportedFlags = RM_GetKeyspaceNotificationFlagsAll(); * if (supportedFlags & REDISMODULE_NOTIFY_LOADED) { * // REDISMODULE_NOTIFY_LOADED is supported * } else{ @@ -8150,8 +8743,8 @@ int RM_ModuleTypeReplaceValue(RedisModuleKey *key, moduleType *mt, void *new_val * an error condition. Error conditions are indicated by setting errno * as folllows: * - * ENOENT: Specified command does not exist. - * EINVAL: Invalid command arity specified. + * * ENOENT: Specified command does not exist. + * * EINVAL: Invalid command arity specified. * * NOTE: The returned array is not a Redis Module object so it does not * get automatically freed even when auto-memory is used. The caller @@ -8247,11 +8840,11 @@ int RM_DefragShouldStop(RedisModuleDefragCtx *ctx) { * data type. * * This behavior is reserved to cases where late defrag is performed. Late - * defrag is selected for keys that implement the free_effort callback and - * return a free_effort value that is larger than the defrag + * defrag is selected for keys that implement the `free_effort` callback and + * return a `free_effort` value that is larger than the defrag * 'active-defrag-max-scan-fields' configuration directive. * - * Smaller keys, keys that do not implement free_effort or the global + * Smaller keys, keys that do not implement `free_effort` or the global * defrag callback are not called in late-defrag mode. In those cases, a * call to this function will return REDISMODULE_ERR. * @@ -8273,7 +8866,7 @@ int RM_DefragCursorSet(RedisModuleDefragCtx *ctx, unsigned long cursor) { /* Fetch a cursor value that has been previously stored using RM_DefragCursorSet(). * * If not called for a late defrag operation, REDISMODULE_ERR will be returned and - * the cursor should be ignored. See DM_DefragCursorSet() for more details on + * the cursor should be ignored. See RM_DefragCursorSet() for more details on * defrag cursors. */ int RM_DefragCursorGet(RedisModuleDefragCtx *ctx, unsigned long *cursor) { @@ -8445,6 +9038,7 @@ void moduleRegisterCoreAPI(void) { REGISTER_API(StringToLongLong); REGISTER_API(StringToDouble); REGISTER_API(StringToLongDouble); + REGISTER_API(StringToStreamID); REGISTER_API(Call); REGISTER_API(CallReplyProto); REGISTER_API(FreeCallReply); @@ -8459,6 +9053,7 @@ void moduleRegisterCoreAPI(void) { REGISTER_API(CreateStringFromDouble); REGISTER_API(CreateStringFromLongDouble); REGISTER_API(CreateStringFromString); + REGISTER_API(CreateStringFromStreamID); REGISTER_API(CreateStringPrintf); REGISTER_API(FreeString); REGISTER_API(StringPtrLen); @@ -8490,6 +9085,15 @@ void moduleRegisterCoreAPI(void) { REGISTER_API(ZsetRangeEndReached); REGISTER_API(HashSet); REGISTER_API(HashGet); + REGISTER_API(StreamAdd); + REGISTER_API(StreamDelete); + REGISTER_API(StreamIteratorStart); + REGISTER_API(StreamIteratorStop); + REGISTER_API(StreamIteratorNextID); + REGISTER_API(StreamIteratorNextField); + REGISTER_API(StreamIteratorDelete); + REGISTER_API(StreamTrimByLength); + REGISTER_API(StreamTrimByID); REGISTER_API(IsKeysPositionRequest); REGISTER_API(KeyAtPos); REGISTER_API(GetClientId); @@ -8539,6 +9143,8 @@ void moduleRegisterCoreAPI(void) { REGISTER_API(GetBlockedClientPrivateData); REGISTER_API(AbortBlock); REGISTER_API(Milliseconds); + REGISTER_API(BlockedClientMeasureTimeStart); + REGISTER_API(BlockedClientMeasureTimeEnd); REGISTER_API(GetThreadSafeContext); REGISTER_API(GetDetachedThreadSafeContext); REGISTER_API(FreeThreadSafeContext); diff --git a/src/modules/gendoc.rb b/src/modules/gendoc.rb index ee6572884..2fd2ec5d7 100644 --- a/src/modules/gendoc.rb +++ b/src/modules/gendoc.rb @@ -4,16 +4,26 @@ # Convert the C comment to markdown def markdown(s) s = s.gsub(/\*\/$/,"") - s = s.gsub(/^ \* {0,1}/,"") - s = s.gsub(/^\/\* /,"") + s = s.gsub(/^ ?\* ?/,"") + s = s.gsub(/^\/\*\*? ?/,"") s.chop! while s[-1] == "\n" || s[-1] == " " lines = s.split("\n") newlines = [] + # Fix some markdown, except in code blocks indented by 4 spaces. lines.each{|l| - if l[0] != ' ' - l = l.gsub(/RM_[A-z()]+/){|x| "`#{x}`"} - l = l.gsub(/RedisModule_[A-z()]+/){|x| "`#{x}`"} - l = l.gsub(/REDISMODULE_[A-z]+/){|x| "`#{x}`"} + if not l.start_with?(' ') + # Rewrite RM_Xyz() to `RedisModule_Xyz()`. The () suffix is + # optional. Even RM_Xyz*() with * as wildcard is handled. + l = l.gsub(/(?\n\n" src = File.open("../module.c").to_a src.each_with_index{|line,i| if line =~ /RM_/ && line[0] != ' ' && line[0] != '#' && line[0] != '/' diff --git a/src/networking.c b/src/networking.c index e624dd8f9..da611675c 100644 --- a/src/networking.c +++ b/src/networking.c @@ -1104,6 +1104,7 @@ void acceptTcpHandler(aeEventLoop *el, int fd, void *privdata, int mask) { "Accepting client connection: %s", server.neterr); return; } + anetCloexec(cfd); serverLog(LL_VERBOSE,"Accepted %s:%d", cip, cport); acceptCommonHandler(connCreateAcceptedSocket(cfd),0,cip); } @@ -1124,6 +1125,7 @@ void acceptTLSHandler(aeEventLoop *el, int fd, void *privdata, int mask) { "Accepting client connection: %s", server.neterr); return; } + anetCloexec(cfd); serverLog(LL_VERBOSE,"Accepted %s:%d", cip, cport); acceptCommonHandler(connCreateAcceptedTLS(cfd, server.tls_auth_clients),0,cip); } @@ -1143,6 +1145,7 @@ void acceptUnixHandler(aeEventLoop *el, int fd, void *privdata, int mask) { "Accepting client connection: %s", server.neterr); return; } + anetCloexec(cfd); serverLog(LL_VERBOSE,"Accepted connection to %s", server.unixsocket); acceptCommonHandler(connCreateAcceptedSocket(cfd),CLIENT_UNIX_SOCKET,NULL); } @@ -1707,7 +1710,7 @@ int processInlineBuffer(client *c) { } /* Handle the \r\n case. */ - if (newline && newline != c->querybuf+c->qb_pos && *(newline-1) == '\r') + if (newline != c->querybuf+c->qb_pos && *(newline-1) == '\r') newline--, linefeed_chars++; /* Split the input buffer up to the \r\n */ @@ -2436,8 +2439,10 @@ void clientCommand(client *c) { " Kill connection made from .", "KILL