diff --git a/.github/workflows/daily.yml b/.github/workflows/daily.yml index 59d236d9a..ee9ac1bbf 100644 --- a/.github/workflows/daily.yml +++ b/.github/workflows/daily.yml @@ -17,7 +17,7 @@ jobs: steps: - uses: actions/checkout@v2 - name: make - run: make + run: make REDIS_CFLAGS='-Werror -DREDIS_TEST' - name: test run: | sudo apt-get install tcl8.6 @@ -28,6 +28,8 @@ jobs: run: ./runtest-sentinel - name: cluster tests run: ./runtest-cluster + - name: unittest + run: ./src/redis-server test all test-ubuntu-libc-malloc: runs-on: ubuntu-latest @@ -76,7 +78,7 @@ jobs: - name: make run: | sudo apt-get update && sudo apt-get install libc6-dev-i386 - make 32bit + make 32bit REDIS_CFLAGS='-Werror -DREDIS_TEST' - name: test run: | sudo apt-get install tcl8.6 @@ -89,6 +91,8 @@ jobs: run: ./runtest-sentinel - name: cluster tests run: ./runtest-cluster + - name: unittest + run: ./src/redis-server test all test-ubuntu-tls: runs-on: ubuntu-latest @@ -142,7 +146,7 @@ jobs: steps: - uses: actions/checkout@v2 - name: make - run: make valgrind + run: make valgrind REDIS_CFLAGS='-Werror -DREDIS_TEST' - name: test run: | sudo apt-get update @@ -150,6 +154,10 @@ jobs: ./runtest --valgrind --verbose --clients 1 --dump-logs - name: module api test run: ./runtest-moduleapi --valgrind --no-latency --verbose --clients 1 + - name: unittest + run: | + valgrind --track-origins=yes --suppressions=./src/valgrind.sup --show-reachable=no --show-possibly-lost=no --leak-check=full --log-file=err.txt ./src/redis-server test all + if grep -q 0x err.txt; then cat err.txt; exit 1; fi test-valgrind-no-malloc-usable-size: runs-on: ubuntu-latest @@ -259,6 +267,7 @@ jobs: test-alpine-jemalloc: runs-on: ubuntu-latest + if: github.repository == 'redis/redis' container: alpine:latest steps: - uses: actions/checkout@v2 @@ -279,6 +288,7 @@ jobs: test-alpine-libc-malloc: runs-on: ubuntu-latest + if: github.repository == 'redis/redis' container: alpine:latest steps: - uses: actions/checkout@v2 diff --git a/README.md b/README.md index d892f4881..87225b34b 100644 --- a/README.md +++ b/README.md @@ -15,10 +15,10 @@ Another good example is to think of Redis as a more complex version of memcached If you want to know more, this is a list of selected starting points: -* Introduction to Redis data types. http://redis.io/topics/data-types-intro +* Introduction to Redis data types. https://redis.io/topics/data-types-intro * Try Redis directly inside your browser. http://try.redis.io -* The full list of Redis commands. http://redis.io/commands -* There is much more inside the official Redis documentation. http://redis.io/documentation +* The full list of Redis commands. https://redis.io/commands +* There is much more inside the official Redis documentation. https://redis.io/documentation Building Redis -------------- @@ -49,7 +49,7 @@ To append a suffix to Redis program names, use: % make PROG_SUFFIX="-alt" -You can run a 32 bit Redis binary using: +You can build a 32 bit Redis binary using: % make 32bit @@ -184,7 +184,7 @@ then in another terminal try the following: (integer) 2 redis> -You can find the list of all the available commands at http://redis.io/commands. +You can find the list of all the available commands at https://redis.io/commands. Installing Redis ----------------- @@ -294,19 +294,19 @@ the structure definition. Another important Redis data structure is the one defining a client. In the past it was called `redisClient`, now just `client`. The structure has many fields, here we'll just show the main ones: - - struct client { - int fd; - sds querybuf; - int argc; - robj **argv; - redisDb *db; - int flags; - list *reply; - char buf[PROTO_REPLY_CHUNK_BYTES]; - ... many other fields ... - } - +```c +struct client { + int fd; + sds querybuf; + int argc; + robj **argv; + redisDb *db; + int flags; + list *reply; + char buf[PROTO_REPLY_CHUNK_BYTES]; + // ... many other fields ... +} +``` The client structure defines a *connected client*: * The `fd` field is the client socket file descriptor. @@ -453,7 +453,7 @@ Other C files * `scripting.c` implements Lua scripting. It is completely self-contained and isolated from the rest of the Redis implementation and is simple enough to understand if you are familiar with the Lua API. * `cluster.c` implements the Redis Cluster. Probably a good read only after being very familiar with the rest of the Redis code base. If you want to read `cluster.c` make sure to read the [Redis Cluster specification][3]. -[3]: http://redis.io/topics/cluster-spec +[3]: https://redis.io/topics/cluster-spec Anatomy of a Redis command --- diff --git a/redis.conf b/redis.conf index 465d56fc0..e8eff2774 100644 --- a/redis.conf +++ b/redis.conf @@ -150,6 +150,11 @@ tcp-keepalive 300 # # tls-cert-file redis.crt # tls-key-file redis.key +# +# If the key file is encrypted using a passphrase, it can be included here +# as well. +# +# tls-key-file-pass secret # Normally Redis uses the same certificate for both server functions (accepting # connections) and client functions (replicating from a master, establishing @@ -162,6 +167,11 @@ tcp-keepalive 300 # # tls-client-cert-file client.crt # tls-client-key-file client.key +# +# If the key file is encrypted using a passphrase, it can be included here +# as well. +# +# tls-client-key-file-pass secret # Configure a DH parameters file to enable Diffie-Hellman (DH) key exchange: # @@ -657,6 +667,18 @@ repl-disable-tcp-nodelay no # By default the priority is 100. replica-priority 100 +# ----------------------------------------------------------------------------- +# By default, Redis Sentinel includes all replicas in its reports. A replica +# can be excluded from Redis Sentinel's announcements. An unannounced replica +# will be ignored by the 'sentinel replicas ' command and won't be +# exposed to Redis Sentinel's clients. +# +# This option does not change the behavior of replica-priority. Even with +# replica-announced set to 'no', the replica can be promoted to master. To +# prevent this behavior, set replica-priority to 0. +# +# replica-announced yes + # It is possible for a master to stop accepting writes if there are less than # N replicas connected, having a lag less or equal than M seconds. # @@ -895,7 +917,7 @@ acllog-max-len 128 # order to provide better out-of-the-box Pub/Sub security. Therefore, it is # recommended that you explicitly define Pub/Sub permissions for all users # rather then rely on implicit default values. Once you've set explicit -# Pub/Sub for all exisitn users, you should uncomment the following line. +# Pub/Sub for all existing users, you should uncomment the following line. # # acl-pubsub-default resetchannels @@ -1225,7 +1247,7 @@ disable-thp yes # If the AOF is enabled on startup Redis will load the AOF, that is the file # with the better durability guarantees. # -# Please check http://redis.io/topics/persistence for more information. +# Please check https://redis.io/topics/persistence for more information. appendonly no @@ -1434,12 +1456,21 @@ lua-time-limit 5000 # master in your cluster. # # Default is 1 (replicas migrate only if their masters remain with at least -# one replica). To disable migration just set it to a very large value. +# one replica). To disable migration just set it to a very large value or +# set cluster-allow-replica-migration to 'no'. # A value of 0 can be set but is useful only for debugging and dangerous # in production. # # cluster-migration-barrier 1 +# Turning off this option allows to use less automatic cluster configuration. +# It both disables migration to orphaned masters and migration from masters +# that became empty. +# +# Default is 'yes' (allow automatic migrations). +# +# cluster-allow-replica-migration yes + # By default Redis Cluster nodes stop accepting queries if they detect there # is at least a hash slot uncovered (no available node is serving it). # This way if the cluster is partially down (for example a range of hash slots @@ -1480,7 +1511,7 @@ lua-time-limit 5000 # cluster-allow-reads-when-down no # In order to setup your cluster make sure to read the documentation -# available at http://redis.io web site. +# available at https://redis.io web site. ########################## CLUSTER DOCKER/NAT support ######################## @@ -1490,16 +1521,21 @@ lua-time-limit 5000 # # In order to make Redis Cluster working in such environments, a static # configuration where each node knows its public address is needed. The -# following two options are used for this scope, and are: +# following four options are used for this scope, and are: # # * cluster-announce-ip # * cluster-announce-port +# * cluster-announce-tls-port # * cluster-announce-bus-port # -# Each instructs the node about its address, client port, and cluster message -# bus port. The information is then published in the header of the bus packets -# so that other nodes will be able to correctly map the address of the node -# publishing the information. +# Each instructs the node about its address, client ports (for connections +# without and with TLS) and cluster message bus port. The information is then +# published in the header of the bus packets so that other nodes will be able to +# correctly map the address of the node publishing the information. +# +# If cluster-tls is set to yes and cluster-announce-tls-port is omitted or set +# to zero, then cluster-announce-port refers to the TLS port. Note also that +# cluster-announce-tls-port has no effect if cluster-tls is set to no. # # If the above options are not used, the normal Redis Cluster auto-detection # will be used instead. @@ -1512,7 +1548,8 @@ lua-time-limit 5000 # Example: # # cluster-announce-ip 10.1.1.5 -# cluster-announce-port 6379 +# cluster-announce-tls-port 6379 +# cluster-announce-port 0 # cluster-announce-bus-port 6380 ################################## SLOW LOG ################################### @@ -1563,7 +1600,7 @@ latency-monitor-threshold 0 ############################# EVENT NOTIFICATION ############################## # Redis can notify Pub/Sub clients about events happening in the key space. -# This feature is documented at http://redis.io/topics/notifications +# This feature is documented at https://redis.io/topics/notifications # # For instance if keyspace events notification is enabled, and a client # performs a DEL operation on key "foo" stored in the Database 0, two @@ -1586,8 +1623,9 @@ latency-monitor-threshold 0 # x Expired events (events generated every time a key expires) # e Evicted events (events generated when a key is evicted for maxmemory) # t Stream commands +# d Module key type events # m Key-miss events (Note: It is not included in the 'A' class) -# A Alias for g$lshzxet, so that the "AKE" string means all the events +# A Alias for g$lshzxetd, so that the "AKE" string means all the events # (Except key-miss events which are excluded from 'A' due to their # unique nature). # diff --git a/src/Makefile b/src/Makefile index ecd69295f..28d50da02 100644 --- a/src/Makefile +++ b/src/Makefile @@ -256,6 +256,17 @@ endif FINAL_LIBS += ../deps/hiredis/libhiredis_ssl.a $(LIBSSL_LIBS) $(LIBCRYPTO_LIBS) endif +ifndef V + define MAKE_INSTALL + @printf ' %b %b\n' $(LINKCOLOR)INSTALL$(ENDCOLOR) $(BINCOLOR)$(1)$(ENDCOLOR) 1>&2 + @$(INSTALL) $(1) $(2) + endef +else + define MAKE_INSTALL + $(INSTALL) $(1) $(2) + endef +endif + REDIS_CC=$(QUIET_CC)$(CC) $(FINAL_CFLAGS) REDIS_LD=$(QUIET_LINK)$(CC) $(FINAL_LDFLAGS) REDIS_INSTALL=$(QUIET_INSTALL)$(INSTALL) @@ -351,9 +362,6 @@ $(REDIS_CLI_NAME): $(REDIS_CLI_OBJ) $(REDIS_BENCHMARK_NAME): $(REDIS_BENCHMARK_OBJ) $(REDIS_LD) -o $@ $^ ../deps/hiredis/libhiredis.a ../deps/hdr_histogram/hdr_histogram.o $(FINAL_LIBS) -dict-benchmark: dict.c zmalloc.c sds.c siphash.c mt19937-64.c - $(REDIS_CC) $(FINAL_CFLAGS) $^ -D DICT_BENCHMARK_MAIN -o $@ $(FINAL_LIBS) - DEP = $(REDIS_SERVER_OBJ:%.o=%.d) $(REDIS_CLI_OBJ:%.o=%.d) $(REDIS_BENCHMARK_OBJ:%.o=%.d) -include $(DEP) @@ -364,7 +372,7 @@ DEP = $(REDIS_SERVER_OBJ:%.o=%.d) $(REDIS_CLI_OBJ:%.o=%.d) $(REDIS_BENCHMARK_OBJ $(REDIS_CC) -MMD -o $@ -c $< clean: - rm -rf $(REDIS_SERVER_NAME) $(REDIS_SENTINEL_NAME) $(REDIS_CLI_NAME) $(REDIS_BENCHMARK_NAME) $(REDIS_CHECK_RDB_NAME) $(REDIS_CHECK_AOF_NAME) *.o *.gcda *.gcno *.gcov redis.info lcov-html Makefile.dep dict-benchmark + rm -rf $(REDIS_SERVER_NAME) $(REDIS_SENTINEL_NAME) $(REDIS_CLI_NAME) $(REDIS_BENCHMARK_NAME) $(REDIS_CHECK_RDB_NAME) $(REDIS_CHECK_AOF_NAME) *.o *.gcda *.gcno *.gcov redis.info lcov-html Makefile.dep rm -f $(DEP) .PHONY: clean @@ -417,9 +425,9 @@ src/help.h: install: all @mkdir -p $(INSTALL_BIN) - $(REDIS_INSTALL) $(REDIS_SERVER_NAME) $(INSTALL_BIN) - $(REDIS_INSTALL) $(REDIS_BENCHMARK_NAME) $(INSTALL_BIN) - $(REDIS_INSTALL) $(REDIS_CLI_NAME) $(INSTALL_BIN) + $(call MAKE_INSTALL,$(REDIS_SERVER_NAME),$(INSTALL_BIN)) + $(call MAKE_INSTALL,$(REDIS_BENCHMARK_NAME),$(INSTALL_BIN)) + $(call MAKE_INSTALL,$(REDIS_CLI_NAME),$(INSTALL_BIN)) @ln -sf $(REDIS_SERVER_NAME) $(INSTALL_BIN)/$(REDIS_CHECK_RDB_NAME) @ln -sf $(REDIS_SERVER_NAME) $(INSTALL_BIN)/$(REDIS_CHECK_AOF_NAME) @ln -sf $(REDIS_SERVER_NAME) $(INSTALL_BIN)/$(REDIS_SENTINEL_NAME) diff --git a/src/acl.c b/src/acl.c index 445409ecd..6a2ade646 100644 --- a/src/acl.c +++ b/src/acl.c @@ -245,7 +245,7 @@ user *ACLCreateUser(const char *name, size_t namelen) { if (raxFind(Users,(unsigned char*)name,namelen) != raxNotFound) return NULL; user *u = zmalloc(sizeof(*u)); u->name = sdsnewlen(name,namelen); - u->flags = USER_FLAG_DISABLED | server.acl_pubusub_default; + u->flags = USER_FLAG_DISABLED | server.acl_pubsub_default; u->allowed_subcommands = NULL; u->passwords = listCreate(); u->patterns = listCreate(); @@ -652,6 +652,7 @@ sds ACLDescribeUser(user *u) { if (u->flags & USER_FLAG_ALLCHANNELS) { res = sdscatlen(res,"&* ",3); } else { + res = sdscatlen(res,"resetchannels ",14); listRewind(u->channels,&li); while((ln = listNext(&li))) { sds thispat = listNodeValue(ln); @@ -1000,6 +1001,8 @@ int ACLSetUser(user *u, const char *op, ssize_t oplen) { serverAssert(ACLSetUser(u,"resetpass",-1) == C_OK); serverAssert(ACLSetUser(u,"resetkeys",-1) == C_OK); serverAssert(ACLSetUser(u,"resetchannels",-1) == C_OK); + if (server.acl_pubsub_default & USER_FLAG_ALLCHANNELS) + serverAssert(ACLSetUser(u,"allchannels",-1) == C_OK); serverAssert(ACLSetUser(u,"off",-1) == C_OK); serverAssert(ACLSetUser(u,"sanitize-payload",-1) == C_OK); serverAssert(ACLSetUser(u,"-@all",-1) == C_OK); @@ -1180,9 +1183,9 @@ int ACLCheckCommandPerm(client *c, int *keyidxptr) { /* If there is no associated user, the connection can run anything. */ if (u == NULL) return ACL_OK; - /* Check if the user can execute this command. */ - if (!(u->flags & USER_FLAG_ALLCOMMANDS) && - c->cmd->proc != authCommand) + /* Check if the user can execute this command or if the command + * doesn't need to be authenticated (hello, auth). */ + if (!(u->flags & USER_FLAG_ALLCOMMANDS) && !(c->cmd->flags & CMD_NO_AUTH)) { /* If the bit is not set we have to check further, in case the * command is allowed just with that specific subcommand. */ @@ -1360,6 +1363,22 @@ int ACLCheckPubsubPerm(client *c, int idx, int count, int literal, int *idxptr) } +/* Check whether the command is ready to be exceuted by ACLCheckCommandPerm. + * If check passes, then check whether pub/sub channels of the command is + * ready to be executed by ACLCheckPubsubPerm */ +int ACLCheckAllPerm(client *c, int *idxptr) { + int acl_retval = ACLCheckCommandPerm(c,idxptr); + if (acl_retval != ACL_OK) + return acl_retval; + if (c->cmd->proc == publishCommand) + acl_retval = ACLCheckPubsubPerm(c,1,1,0,idxptr); + else if (c->cmd->proc == subscribeCommand) + acl_retval = ACLCheckPubsubPerm(c,1,c->argc-1,0,idxptr); + else if (c->cmd->proc == psubscribeCommand) + acl_retval = ACLCheckPubsubPerm(c,1,c->argc-1,1,idxptr); + return acl_retval; +} + /* ============================================================================= * ACL loading / saving functions * ==========================================================================*/ @@ -1873,6 +1892,10 @@ void addACLLogEntry(client *c, int reason, int argpos, sds username) { void aclCommand(client *c) { char *sub = c->argv[1]->ptr; if (!strcasecmp(sub,"setuser") && c->argc >= 3) { + /* Consider information about passwords or permissions + * to be sensitive, which will be the arguments for this + * subcommand. */ + preventCommandLogging(c); sds username = c->argv[2]->ptr; /* Check username validity. */ if (ACLStringHasSpaces(username,sdslen(username))) { diff --git a/src/ae.c b/src/ae.c index 283f51438..48fb63ed9 100644 --- a/src/ae.c +++ b/src/ae.c @@ -239,7 +239,7 @@ int aeDeleteTimeEvent(aeEventLoop *eventLoop, long long id) return AE_ERR; /* NO event with the specified ID found */ } -/* How many milliseconds until the first timer should fire. +/* How many microseconds until the first timer should fire. * If there are no timers, -1 is returned. * * Note that's O(N) since time events are unsorted. @@ -248,7 +248,7 @@ int aeDeleteTimeEvent(aeEventLoop *eventLoop, long long id) * Much better but still insertion or deletion of timers is O(N). * 2) Use a skiplist to have this operation as O(1) and insertion as O(log(N)). */ -static long msUntilEarliestTimer(aeEventLoop *eventLoop) { +static int64_t usUntilEarliestTimer(aeEventLoop *eventLoop) { aeTimeEvent *te = eventLoop->timeEventHead; if (te == NULL) return -1; @@ -260,8 +260,7 @@ static long msUntilEarliestTimer(aeEventLoop *eventLoop) { } monotime now = getMonotonicUs(); - return (now >= earliest->when) - ? 0 : (long)((earliest->when - now) / 1000); + return (now >= earliest->when) ? 0 : earliest->when - now; } /* Process time events */ @@ -361,14 +360,14 @@ int aeProcessEvents(aeEventLoop *eventLoop, int flags) ((flags & AE_TIME_EVENTS) && !(flags & AE_DONT_WAIT))) { int j; struct timeval tv, *tvp; - long msUntilTimer = -1; + int64_t usUntilTimer = -1; if (flags & AE_TIME_EVENTS && !(flags & AE_DONT_WAIT)) - msUntilTimer = msUntilEarliestTimer(eventLoop); + usUntilTimer = usUntilEarliestTimer(eventLoop); - if (msUntilTimer >= 0) { - tv.tv_sec = msUntilTimer / 1000; - tv.tv_usec = (msUntilTimer % 1000) * 1000; + if (usUntilTimer >= 0) { + tv.tv_sec = usUntilTimer / 1000000; + tv.tv_usec = usUntilTimer % 1000000; tvp = &tv; } else { /* If we have to check for events but need to return diff --git a/src/ae_epoll.c b/src/ae_epoll.c index 07ca8ca41..023c93a17 100644 --- a/src/ae_epoll.c +++ b/src/ae_epoll.c @@ -111,7 +111,7 @@ static int aeApiPoll(aeEventLoop *eventLoop, struct timeval *tvp) { int retval, numevents = 0; retval = epoll_wait(state->epfd,state->events,eventLoop->setsize, - tvp ? (tvp->tv_sec*1000 + tvp->tv_usec/1000) : -1); + tvp ? (tvp->tv_sec*1000 + (tvp->tv_usec + 999)/1000) : -1); if (retval > 0) { int j; diff --git a/src/anet.c b/src/anet.c index 0bfa575f5..a121c2768 100644 --- a/src/anet.c +++ b/src/anet.c @@ -186,27 +186,6 @@ int anetDisableTcpNoDelay(char *err, int fd) return anetSetTcpNoDelay(err, fd, 0); } - -int anetSetSendBuffer(char *err, int fd, int buffsize) -{ - if (setsockopt(fd, SOL_SOCKET, SO_SNDBUF, &buffsize, sizeof(buffsize)) == -1) - { - anetSetError(err, "setsockopt SO_SNDBUF: %s", strerror(errno)); - return ANET_ERR; - } - return ANET_OK; -} - -int anetTcpKeepAlive(char *err, int fd) -{ - int yes = 1; - if (setsockopt(fd, SOL_SOCKET, SO_KEEPALIVE, &yes, sizeof(yes)) == -1) { - anetSetError(err, "setsockopt SO_KEEPALIVE: %s", strerror(errno)); - return ANET_ERR; - } - return ANET_OK; -} - /* Set the socket send timeout (SO_SNDTIMEO socket option) to the specified * number of milliseconds, or disable it if the 'ms' argument is zero. */ int anetSendTimeout(char *err, int fd, long long ms) { @@ -378,23 +357,11 @@ end: } } -int anetTcpConnect(char *err, const char *addr, int port) -{ - return anetTcpGenericConnect(err,addr,port,NULL,ANET_CONNECT_NONE); -} - int anetTcpNonBlockConnect(char *err, const char *addr, int port) { return anetTcpGenericConnect(err,addr,port,NULL,ANET_CONNECT_NONBLOCK); } -int anetTcpNonBlockBindConnect(char *err, const char *addr, int port, - const char *source_addr) -{ - return anetTcpGenericConnect(err,addr,port,source_addr, - ANET_CONNECT_NONBLOCK); -} - int anetTcpNonBlockBestEffortBindConnect(char *err, const char *addr, int port, const char *source_addr) { @@ -430,46 +397,6 @@ int anetUnixGenericConnect(char *err, const char *path, int flags) return s; } -int anetUnixConnect(char *err, const char *path) -{ - return anetUnixGenericConnect(err,path,ANET_CONNECT_NONE); -} - -int anetUnixNonBlockConnect(char *err, const char *path) -{ - return anetUnixGenericConnect(err,path,ANET_CONNECT_NONBLOCK); -} - -/* Like read(2) but make sure 'count' is read before to return - * (unless error or EOF condition is encountered) */ -int anetRead(int fd, char *buf, int count) -{ - ssize_t nread, totlen = 0; - while(totlen != count) { - nread = read(fd,buf,count-totlen); - if (nread == 0) return totlen; - if (nread == -1) return -1; - totlen += nread; - buf += nread; - } - return totlen; -} - -/* Like write(2) but make sure 'count' is written before to return - * (unless error is encountered) */ -int anetWrite(int fd, char *buf, int count) -{ - ssize_t nwritten, totlen = 0; - while(totlen != count) { - nwritten = write(fd,buf,count-totlen); - if (nwritten == 0) return totlen; - if (nwritten == -1) return -1; - totlen += nwritten; - buf += nwritten; - } - return totlen; -} - static int anetListen(char *err, int s, struct sockaddr *sa, socklen_t len, int backlog) { if (bind(s,sa,len) == -1) { anetSetError(err, "bind: %s", strerror(errno)); diff --git a/src/anet.h b/src/anet.h index 5da2f3b46..2a685cc01 100644 --- a/src/anet.h +++ b/src/anet.h @@ -53,26 +53,19 @@ #define FD_TO_PEER_NAME 0 #define FD_TO_SOCK_NAME 1 -int anetTcpConnect(char *err, const char *addr, int port); int anetTcpNonBlockConnect(char *err, const char *addr, int port); -int anetTcpNonBlockBindConnect(char *err, const char *addr, int port, const char *source_addr); int anetTcpNonBlockBestEffortBindConnect(char *err, const char *addr, int port, const char *source_addr); -int anetUnixConnect(char *err, const char *path); -int anetUnixNonBlockConnect(char *err, const char *path); -int anetRead(int fd, char *buf, int count); int anetResolve(char *err, char *host, char *ipbuf, size_t ipbuf_len, int flags); int anetTcpServer(char *err, int port, char *bindaddr, int backlog); int anetTcp6Server(char *err, int port, char *bindaddr, int backlog); int anetUnixServer(char *err, char *path, mode_t perm, int backlog); int anetTcpAccept(char *err, int serversock, char *ip, size_t ip_len, int *port); int anetUnixAccept(char *err, int serversock); -int anetWrite(int fd, char *buf, int count); int anetNonBlock(char *err, int fd); int anetBlock(char *err, int fd); int anetCloexec(int fd); int anetEnableTcpNoDelay(char *err, int fd); int anetDisableTcpNoDelay(char *err, int fd); -int anetTcpKeepAlive(char *err, int fd); int anetSendTimeout(char *err, int fd, long long ms); int anetRecvTimeout(char *err, int fd, long long ms); int anetFdToString(int fd, char *ip, size_t ip_len, int *port, int fd_to_str_type); diff --git a/src/aof.c b/src/aof.c index f1586cf90..0d2971eac 100644 --- a/src/aof.c +++ b/src/aof.c @@ -218,7 +218,7 @@ void killAppendOnlyChild(void) { serverLog(LL_NOTICE,"Killing running AOF rewrite child: %ld", (long) server.child_pid); if (kill(server.child_pid,SIGUSR1) != -1) { - while(wait3(&statloc,0,NULL) != server.child_pid); + while(waitpid(-1, &statloc, 0) != server.child_pid); } /* Reset the buffer accumulating changes while the child saves. */ aofRewriteBufferReset(); @@ -234,9 +234,12 @@ void killAppendOnlyChild(void) { void stopAppendOnly(void) { serverAssert(server.aof_state != AOF_OFF); flushAppendOnlyFile(1); - redis_fsync(server.aof_fd); - server.aof_fsync_offset = server.aof_current_size; - server.aof_last_fsync = server.unixtime; + if (redis_fsync(server.aof_fd) == -1) { + serverLog(LL_WARNING,"Fail to fsync the AOF file: %s",strerror(errno)); + } else { + server.aof_fsync_offset = server.aof_current_size; + server.aof_last_fsync = server.unixtime; + } close(server.aof_fd); server.aof_fd = -1; @@ -290,6 +293,15 @@ int startAppendOnly(void) { server.aof_last_fsync = server.unixtime; server.aof_fd = newfd; + /* If AOF fsync error in bio job, we just ignore it and log the event. */ + int aof_bio_fsync_status; + atomicGet(server.aof_bio_fsync_status, aof_bio_fsync_status); + if (aof_bio_fsync_status == C_ERR) { + serverLog(LL_WARNING, + "AOF reopen, just ignore the AOF fsync error in bio job"); + atomicSet(server.aof_bio_fsync_status,C_OK); + } + /* If AOF was in error state, we just ignore it and log the event. */ if (server.aof_last_write_status == C_ERR) { serverLog(LL_WARNING,"AOF reopen, just ignore the last error."); @@ -1590,7 +1602,7 @@ int rewriteAppendOnlyFile(char *filename) { if (write(server.aof_pipe_write_ack_to_parent,"!",1) != 1) goto werr; if (anetNonBlock(NULL,server.aof_pipe_read_ack_from_parent) != ANET_OK) goto werr; - /* We read the ACK from the server using a 10 seconds timeout. Normally + /* We read the ACK from the server using a 5 seconds timeout. Normally * it should reply ASAP, but just in case we lose its reply, we are sure * the child will eventually get terminated. */ if (syncRead(server.aof_pipe_read_ack_from_parent,&byte,1,5000) != 1 || diff --git a/src/asciilogo.h b/src/asciilogo.h index 044ca0c55..a62f68cf9 100644 --- a/src/asciilogo.h +++ b/src/asciilogo.h @@ -31,13 +31,13 @@ const char *ascii_logo = " _._ \n" " _.-``__ ''-._ \n" " _.-`` `. `_. ''-._ Redis %s (%s/%d) %s bit\n" -" .-`` .-```. ```\\/ _.,_ ''-._ \n" +" .-`` .-```. ```\\/ _.,_ ''-._ \n" " ( ' , .-` | `, ) Running in %s mode\n" " |`-._`-...-` __...-.``-._|'` _.-'| Port: %d\n" " | `-._ `._ / _.-' | PID: %ld\n" " `-._ `-._ `-./ _.-' _.-' \n" " |`-._`-._ `-.__.-' _.-'_.-'| \n" -" | `-._`-._ _.-'_.-' | http://redis.io \n" +" | `-._`-._ _.-'_.-' | https://redis.io \n" " `-._ `-._`-.__.-'_.-' _.-' \n" " |`-._`-._ `-.__.-' _.-'_.-'| \n" " | `-._`-._ _.-'_.-' | \n" diff --git a/src/bio.c b/src/bio.c index c6e17f49d..ff18c4fe7 100644 --- a/src/bio.c +++ b/src/bio.c @@ -220,7 +220,23 @@ void *bioProcessBackgroundJobs(void *arg) { if (type == BIO_CLOSE_FILE) { close(job->fd); } else if (type == BIO_AOF_FSYNC) { - redis_fsync(job->fd); + /* The fd may be closed by main thread and reused for another + * socket, pipe, or file. We just ignore these errno because + * aof fsync did not really fail. */ + if (redis_fsync(job->fd) == -1 && + errno != EBADF && errno != EINVAL) + { + int last_status; + atomicGet(server.aof_bio_fsync_status,last_status); + atomicSet(server.aof_bio_fsync_status,C_ERR); + atomicSet(server.aof_bio_fsync_errno,errno); + if (last_status == C_OK) { + serverLog(LL_WARNING, + "Fail to fsync the AOF file: %s",strerror(errno)); + } + } else { + atomicSet(server.aof_bio_fsync_status,C_OK); + } } else if (type == BIO_LAZY_FREE) { job->free_fn(job->free_args); } else { diff --git a/src/blocked.c b/src/blocked.c index 09e17213c..eb110bd35 100644 --- a/src/blocked.c +++ b/src/blocked.c @@ -106,12 +106,11 @@ void blockClient(client *c, int btype) { void updateStatsOnUnblock(client *c, long blocked_us, long reply_us){ const ustime_t total_cmd_duration = c->duration + blocked_us + reply_us; c->lastcmd->microseconds += total_cmd_duration; + /* Log the command into the Slow log if needed. */ - if (!(c->lastcmd->flags & CMD_SKIP_SLOWLOG)) { - slowlogPushEntryIfNeeded(c,c->argv,c->argc,total_cmd_duration); - /* Log the reply duration event. */ - latencyAddSampleIfNeeded("command-unblocking",reply_us/1000); - } + slowlogPushCurrentCommand(c, c->lastcmd, total_cmd_duration); + /* Log the reply duration event. */ + latencyAddSampleIfNeeded("command-unblocking",reply_us/1000); } /* This function is called in the beforeSleep() function of the event loop @@ -188,6 +187,16 @@ void unblockClient(client *c) { } else { serverPanic("Unknown btype in unblockClient()."); } + + /* Reset the client for a new query since, for blocking commands + * we do not do it immediately after the command returns (when the + * client got blocked) in order to be still able to access the argument + * vector from module callbacks and updateStatsOnUnblock. */ + if (c->btype != BLOCKED_PAUSE) { + freeClientOriginalArgv(c); + resetClient(c); + } + /* Clear the flags, and put the client in the unblocked list so that * we'll process new commands in its query buffer ASAP. */ server.blocked_clients--; @@ -279,7 +288,6 @@ void serveClientsBlockedOnListKey(robj *o, readyList *rl) { * freed by the next unblockClient() * call. */ if (dstkey) incrRefCount(dstkey); - unblockClient(receiver); monotime replyTimer; elapsedStart(&replyTimer); @@ -292,6 +300,7 @@ void serveClientsBlockedOnListKey(robj *o, readyList *rl) { listTypePush(o,value,wherefrom); } updateStatsOnUnblock(receiver, 0, elapsedUs(replyTimer)); + unblockClient(receiver); if (dstkey) decrRefCount(dstkey); decrRefCount(value); @@ -335,11 +344,11 @@ void serveClientsBlockedOnSortedSetKey(robj *o, readyList *rl) { int where = (receiver->lastcmd && receiver->lastcmd->proc == bzpopminCommand) ? ZSET_MIN : ZSET_MAX; - unblockClient(receiver); monotime replyTimer; elapsedStart(&replyTimer); genericZpopCommand(receiver,&rl->key,1,where,1,NULL); updateStatsOnUnblock(receiver, 0, elapsedUs(replyTimer)); + unblockClient(receiver); zcard--; /* Replicate the command. */ @@ -471,6 +480,10 @@ void serveClientsBlockedOnStreamKey(robj *o, readyList *rl) { void serveClientsBlockedOnKeyByModule(readyList *rl) { dictEntry *de; + /* Optimization: If no clients are in type BLOCKED_MODULE, + * we can skip this loop. */ + if (!server.blocked_clients_by_type[BLOCKED_MODULE]) return; + /* We serve clients in the same order they blocked for * this key, from the first blocked to the last. */ de = dictFind(rl->db->blocking_keys,rl->key); @@ -553,7 +566,7 @@ void handleClientsBlockedOnKeys(void) { * way we can lookup an object multiple times (BLMOVE does * that) without the risk of it being freed in the second * lookup, invalidating the first one. - * See https://github.com/antirez/redis/pull/6554. */ + * See https://github.com/redis/redis/pull/6554. */ server.fixed_time_expire++; updateCachedTime(0); diff --git a/src/childinfo.c b/src/childinfo.c index e3f33a96c..4f0a42001 100644 --- a/src/childinfo.c +++ b/src/childinfo.c @@ -33,6 +33,7 @@ typedef struct { size_t keys; size_t cow; + monotime cow_updated; double progress; childInfoType information_type; /* Type of information */ } child_info_data; @@ -69,18 +70,39 @@ void closeChildInfoPipe(void) { void sendChildInfoGeneric(childInfoType info_type, size_t keys, double progress, char *pname) { if (server.child_info_pipe[1] == -1) return; - child_info_data data = {0}; /* zero everything, including padding to sattisfy valgrind */ + static monotime cow_updated = 0; + static uint64_t cow_update_cost = 0; + static size_t cow = 0; + + child_info_data data = {0}; /* zero everything, including padding to satisfy valgrind */ + + /* When called to report current info, we need to throttle down CoW updates as they + * can be very expensive. To do that, we measure the time it takes to get a reading + * and schedule the next reading to happen not before time*CHILD_COW_COST_FACTOR + * passes. */ + + monotime now = getMonotonicUs(); + if (info_type != CHILD_INFO_TYPE_CURRENT_INFO || + !cow_updated || + now - cow_updated > cow_update_cost * CHILD_COW_DUTY_CYCLE) + { + cow = zmalloc_get_private_dirty(-1); + cow_updated = getMonotonicUs(); + cow_update_cost = cow_updated - now; + + if (cow) { + serverLog((info_type == CHILD_INFO_TYPE_CURRENT_INFO) ? LL_VERBOSE : LL_NOTICE, + "%s: %zu MB of memory used by copy-on-write", + pname, data.cow / (1024 * 1024)); + } + } + data.information_type = info_type; data.keys = keys; - data.cow = zmalloc_get_private_dirty(-1); + data.cow = cow; + data.cow_updated = cow_updated; data.progress = progress; - if (data.cow) { - serverLog((info_type == CHILD_INFO_TYPE_CURRENT_INFO) ? LL_VERBOSE : LL_NOTICE, - "%s: %zu MB of memory used by copy-on-write", - pname, data.cow/(1024*1024)); - } - ssize_t wlen = sizeof(data); if (write(server.child_info_pipe[1], &data, wlen) != wlen) { @@ -89,9 +111,10 @@ void sendChildInfoGeneric(childInfoType info_type, size_t keys, double progress, } /* Update Child info. */ -void updateChildInfo(childInfoType information_type, size_t cow, size_t keys, double progress) { +void updateChildInfo(childInfoType information_type, size_t cow, monotime cow_updated, size_t keys, double progress) { if (information_type == CHILD_INFO_TYPE_CURRENT_INFO) { server.stat_current_cow_bytes = cow; + server.stat_current_cow_updated = cow_updated; server.stat_current_save_keys_processed = keys; if (progress != -1) server.stat_module_progress = progress; } else if (information_type == CHILD_INFO_TYPE_AOF_COW_SIZE) { @@ -107,7 +130,7 @@ void updateChildInfo(childInfoType information_type, size_t cow, size_t keys, do * if complete data read into the buffer, * data is stored into *buffer, and returns 1. * otherwise, the partial data is left in the buffer, waiting for the next read, and returns 0. */ -int readChildInfo(childInfoType *information_type, size_t *cow, size_t *keys, double* progress) { +int readChildInfo(childInfoType *information_type, size_t *cow, monotime *cow_updated, size_t *keys, double* progress) { /* We are using here a static buffer in combination with the server.child_info_nread to handle short reads */ static child_info_data buffer; ssize_t wlen = sizeof(buffer); @@ -124,6 +147,7 @@ int readChildInfo(childInfoType *information_type, size_t *cow, size_t *keys, do if (server.child_info_nread == wlen) { *information_type = buffer.information_type; *cow = buffer.cow; + *cow_updated = buffer.cow_updated; *keys = buffer.keys; *progress = buffer.progress; return 1; @@ -137,12 +161,13 @@ void receiveChildInfo(void) { if (server.child_info_pipe[0] == -1) return; size_t cow; + monotime cow_updated; size_t keys; double progress; childInfoType information_type; /* Drain the pipe and update child info so that we get the final message. */ - while (readChildInfo(&information_type, &cow, &keys, &progress)) { - updateChildInfo(information_type, cow, keys, progress); + while (readChildInfo(&information_type, &cow, &cow_updated, &keys, &progress)) { + updateChildInfo(information_type, cow, cow_updated, keys, progress); } } diff --git a/src/cluster.c b/src/cluster.c index 4605e3ea9..0f0ab737e 100644 --- a/src/cluster.c +++ b/src/cluster.c @@ -55,7 +55,7 @@ void clusterSendFail(char *nodename); void clusterSendFailoverAuthIfNeeded(clusterNode *node, clusterMsg *request); void clusterUpdateState(void); int clusterNodeGetSlotBit(clusterNode *n, int slot); -sds clusterGenNodesDescription(int filter); +sds clusterGenNodesDescription(int filter, int use_pport); clusterNode *clusterLookupNode(const char *name); int clusterNodeAddSlave(clusterNode *master, clusterNode *slave); int clusterAddSlot(clusterNode *n, int slot); @@ -190,6 +190,9 @@ int clusterLoadConfig(char *filename) { * base port. */ n->cport = busp ? atoi(busp) : n->port + CLUSTER_PORT_INCR; + /* The plaintext port for client in a TLS cluster (n->pport) is not + * stored in nodes.conf. It is received later over the bus protocol. */ + /* Parse flags */ p = s = argv[2]; while(p) { @@ -336,7 +339,7 @@ int clusterSaveConfig(int do_fsync) { /* Get the nodes description and concatenate our "vars" directive to * save currentEpoch and lastVoteEpoch. */ - ci = clusterGenNodesDescription(CLUSTER_NODE_HANDSHAKE); + ci = clusterGenNodesDescription(CLUSTER_NODE_HANDSHAKE, 0); ci = sdscatprintf(ci,"vars currentEpoch %llu lastVoteEpoch %llu\n", (unsigned long long) server.cluster->currentEpoch, (unsigned long long) server.cluster->lastVoteEpoch); @@ -355,7 +358,7 @@ int clusterSaveConfig(int do_fsync) { if (write(fd,ci,sdslen(ci)) != (ssize_t)sdslen(ci)) goto err; if (do_fsync) { server.cluster->todo_before_sleep &= ~CLUSTER_TODO_FSYNC_CONFIG; - fsync(fd); + if (fsync(fd) == -1) goto err; } /* Truncate the file if needed to remove the final \n padding that @@ -437,6 +440,26 @@ int clusterLockConfig(char *filename) { return C_OK; } +/* Derives our ports to be announced in the cluster bus. */ +void deriveAnnouncedPorts(int *announced_port, int *announced_pport, + int *announced_cport) { + int port = server.tls_cluster ? server.tls_port : server.port; + /* Default announced ports. */ + *announced_port = port; + *announced_pport = server.tls_cluster ? server.port : 0; + *announced_cport = port + CLUSTER_PORT_INCR; + /* Config overriding announced ports. */ + if (server.tls_cluster && server.cluster_announce_tls_port) { + *announced_port = server.cluster_announce_tls_port; + *announced_pport = server.cluster_announce_port; + } else if (server.cluster_announce_port) { + *announced_port = server.cluster_announce_port; + } + if (server.cluster_announce_bus_port) { + *announced_cport = server.cluster_announce_bus_port; + } +} + /* Some flags (currently just the NOFAILOVER flag) may need to be updated * in the "myself" node based on the current configuration of the node, * that may change at runtime via CONFIG SET. This function changes the @@ -524,14 +547,9 @@ void clusterInit(void) { memset(server.cluster->slots_keys_count,0, sizeof(server.cluster->slots_keys_count)); - /* Set myself->port / cport to my listening ports, we'll just need to + /* Set myself->port/cport/pport to my listening ports, we'll just need to * discover the IP address via MEET messages. */ - myself->port = port; - myself->cport = port+CLUSTER_PORT_INCR; - if (server.cluster_announce_port) - myself->port = server.cluster_announce_port; - if (server.cluster_announce_bus_port) - myself->cport = server.cluster_announce_bus_port; + deriveAnnouncedPorts(&myself->port, &myself->pport, &myself->cport); server.cluster->mf_end = 0; resetManualFailover(); @@ -782,6 +800,7 @@ clusterNode *createClusterNode(char *nodename, int flags) { memset(node->ip,0,sizeof(node->ip)); node->port = 0; node->cport = 0; + node->pport = 0; node->fail_reports = listCreate(); node->voted_time = 0; node->orphaned_time = 0; @@ -1488,6 +1507,7 @@ void clusterProcessGossipSection(clusterMsg *hdr, clusterLink *link) { if (node->link) freeClusterLink(node->link); memcpy(node->ip,g->ip,NET_IP_STR_LEN); node->port = ntohs(g->port); + node->pport = ntohs(g->pport); node->cport = ntohs(g->cport); node->flags &= ~CLUSTER_NODE_NOADDR; } @@ -1509,6 +1529,7 @@ void clusterProcessGossipSection(clusterMsg *hdr, clusterLink *link) { node = createClusterNode(g->nodename, flags); memcpy(node->ip,g->ip,NET_IP_STR_LEN); node->port = ntohs(g->port); + node->pport = ntohs(g->pport); node->cport = ntohs(g->cport); clusterAddNode(node); } @@ -1548,6 +1569,7 @@ int nodeUpdateAddressIfNeeded(clusterNode *node, clusterLink *link, { char ip[NET_IP_STR_LEN] = {0}; int port = ntohs(hdr->port); + int pport = ntohs(hdr->pport); int cport = ntohs(hdr->cport); /* We don't proceed if the link is the same as the sender link, as this @@ -1559,12 +1581,13 @@ int nodeUpdateAddressIfNeeded(clusterNode *node, clusterLink *link, if (link == node->link) return 0; nodeIp2String(ip,link,hdr->myip); - if (node->port == port && node->cport == cport && + if (node->port == port && node->cport == cport && node->pport == pport && strcmp(ip,node->ip) == 0) return 0; /* IP / port is different, update it. */ memcpy(node->ip,ip,sizeof(ip)); node->port = port; + node->pport = pport; node->cport = cport; if (node->link) freeClusterLink(node->link); node->flags &= ~CLUSTER_NODE_NOADDR; @@ -1610,7 +1633,7 @@ void clusterSetNodeAsMaster(clusterNode *n) { * case we receive the info via an UPDATE packet. */ void clusterUpdateSlotsConfigWith(clusterNode *sender, uint64_t senderConfigEpoch, unsigned char *slots) { int j; - clusterNode *curmaster, *newmaster = NULL; + clusterNode *curmaster = NULL, *newmaster = NULL; /* The dirty slots list is a list of slots for which we lose the ownership * while having still keys inside. This usually happens after a failover * or after a manual cluster reconfiguration operated by the admin. @@ -1621,6 +1644,12 @@ void clusterUpdateSlotsConfigWith(clusterNode *sender, uint64_t senderConfigEpoc uint16_t dirty_slots[CLUSTER_SLOTS]; int dirty_slots_count = 0; + /* We should detect if sender is new master of our shard. + * We will know it if all our slots were migrated to sender, and sender + * has no slots except ours */ + int sender_slots = 0; + int migrated_our_slots = 0; + /* Here we set curmaster to this node or the node this node * replicates to if it's a slave. In the for loop we are * interested to check if slots are taken away from curmaster. */ @@ -1633,6 +1662,8 @@ void clusterUpdateSlotsConfigWith(clusterNode *sender, uint64_t senderConfigEpoc for (j = 0; j < CLUSTER_SLOTS; j++) { if (bitmapTestBit(slots,j)) { + sender_slots++; + /* The slot is already bound to the sender of this message. */ if (server.cluster->slots[j] == sender) continue; @@ -1659,8 +1690,10 @@ void clusterUpdateSlotsConfigWith(clusterNode *sender, uint64_t senderConfigEpoc dirty_slots_count++; } - if (server.cluster->slots[j] == curmaster) + if (server.cluster->slots[j] == curmaster) { newmaster = sender; + migrated_our_slots++; + } clusterDelSlot(j); clusterAddSlot(sender,j); clusterDoBeforeSleep(CLUSTER_TODO_SAVE_CONFIG| @@ -1683,7 +1716,9 @@ void clusterUpdateSlotsConfigWith(clusterNode *sender, uint64_t senderConfigEpoc * master. * 2) We are a slave and our master is left without slots. We need * to replicate to the new slots owner. */ - if (newmaster && curmaster->numslots == 0) { + if (newmaster && curmaster->numslots == 0 && + (server.cluster_allow_replica_migration || + sender_slots == migrated_our_slots)) { serverLog(LL_WARNING, "Configuration change detected. Reconfiguring myself " "as a replica of %.40s", sender->name); @@ -1811,7 +1846,7 @@ int clusterProcessPacket(clusterLink *link) { nodeIsSlave(myself) && myself->slaveof == sender && hdr->mflags[0] & CLUSTERMSG_FLAG0_PAUSED && - server.cluster->mf_master_offset == 0) + server.cluster->mf_master_offset == -1) { server.cluster->mf_master_offset = sender->repl_offset; clusterDoBeforeSleep(CLUSTER_TODO_HANDLE_MANUALFAILOVER); @@ -1862,6 +1897,7 @@ int clusterProcessPacket(clusterLink *link) { node = createClusterNode(NULL,CLUSTER_NODE_HANDSHAKE); nodeIp2String(node->ip,link,hdr->myip); node->port = ntohs(hdr->port); + node->pport = ntohs(hdr->pport); node->cport = ntohs(hdr->cport); clusterAddNode(node); clusterDoBeforeSleep(CLUSTER_TODO_SAVE_CONFIG); @@ -1924,6 +1960,7 @@ int clusterProcessPacket(clusterLink *link) { link->node->flags |= CLUSTER_NODE_NOADDR; link->node->ip[0] = '\0'; link->node->port = 0; + link->node->pport = 0; link->node->cport = 0; freeClusterLink(link); clusterDoBeforeSleep(CLUSTER_TODO_SAVE_CONFIG); @@ -2423,19 +2460,16 @@ void clusterBuildMessageHdr(clusterMsg *hdr, int type) { hdr->myip[NET_IP_STR_LEN-1] = '\0'; } - /* Handle cluster-announce-port as well. */ - int port = server.tls_cluster ? server.tls_port : server.port; - int announced_port = server.cluster_announce_port ? - server.cluster_announce_port : port; - int announced_cport = server.cluster_announce_bus_port ? - server.cluster_announce_bus_port : - (port + CLUSTER_PORT_INCR); + /* Handle cluster-announce-[tls-|bus-]port. */ + int announced_port, announced_pport, announced_cport; + deriveAnnouncedPorts(&announced_port, &announced_pport, &announced_cport); memcpy(hdr->myslots,master->slots,sizeof(hdr->myslots)); memset(hdr->slaveof,0,CLUSTER_NAMELEN); if (myself->slaveof != NULL) memcpy(hdr->slaveof,myself->slaveof->name, CLUSTER_NAMELEN); hdr->port = htons(announced_port); + hdr->pport = htons(announced_pport); hdr->cport = htons(announced_cport); hdr->flags = htons(myself->flags); hdr->state = server.cluster->state; @@ -2492,6 +2526,7 @@ void clusterSetGossipEntry(clusterMsg *hdr, int i, clusterNode *n) { gossip->port = htons(n->port); gossip->cport = htons(n->cport); gossip->flags = htons(n->flags); + gossip->pport = htons(n->pport); gossip->notused1 = 0; } @@ -3090,7 +3125,7 @@ void clusterFailoverReplaceYourMaster(void) { /* This function is called if we are a slave node and our master serving * a non-zero amount of hash slots is in FAIL state. * - * The gaol of this function is: + * The goal of this function is: * 1) To check if we are able to perform a failover, is our data updated? * 2) Try to get elected by masters. * 3) Perform the failover informing all the other nodes. @@ -3135,7 +3170,7 @@ void clusterHandleSlaveFailover(void) { return; } - /* Set data_age to the number of seconds we are disconnected from + /* Set data_age to the number of milliseconds we are disconnected from * the master. */ if (server.repl_state == REPL_STATE_CONNECTED) { data_age = (mstime_t)(server.unixtime - server.master->lastinteraction) @@ -3419,7 +3454,7 @@ void resetManualFailover(void) { server.cluster->mf_end = 0; /* No manual failover in progress. */ server.cluster->mf_can_start = 0; server.cluster->mf_slave = NULL; - server.cluster->mf_master_offset = 0; + server.cluster->mf_master_offset = -1; } /* If a manual failover timed out, abort it. */ @@ -3440,7 +3475,7 @@ void clusterHandleManualFailover(void) { * next steps are performed by clusterHandleSlaveFailover(). */ if (server.cluster->mf_can_start) return; - if (server.cluster->mf_master_offset == 0) return; /* Wait for offset... */ + if (server.cluster->mf_master_offset == -1) return; /* Wait for offset... */ if (server.cluster->mf_master_offset == replicationGetSlaveOffset()) { /* Our replication offset matches the master replication offset @@ -3630,7 +3665,6 @@ void clusterCron(void) { now - node->link->ctime > server.cluster_node_timeout && /* was not already reconnected */ node->ping_sent && /* we already sent a ping */ - node->pong_received < node->ping_sent && /* still waiting pong */ /* and we are waiting for the pong more than timeout/2 */ ping_delay > server.cluster_node_timeout/2 && /* and in such interval we are not seeing any traffic at all. */ @@ -3713,7 +3747,8 @@ void clusterCron(void) { * the orphaned masters. Note that it does not make sense to try * a migration if there is no master with at least *two* working * slaves. */ - if (orphaned_masters && max_slaves >= 2 && this_slaves == max_slaves) + if (orphaned_masters && max_slaves >= 2 && this_slaves == max_slaves && + server.cluster_allow_replica_migration) clusterHandleSlaveMigration(max_slaves); } @@ -3823,7 +3858,7 @@ int clusterNodeSetSlotBit(clusterNode *n, int slot) { * However new masters with slots assigned are considered valid * migration targets if the rest of the cluster is not a slave-less. * - * See https://github.com/antirez/redis/issues/3043 for more info. */ + * See https://github.com/redis/redis/issues/3043 for more info. */ if (n->numslots == 1 && clusterMastersHaveSlaves()) n->flags |= CLUSTER_NODE_MIGRATE_TO; } @@ -4131,15 +4166,16 @@ sds representClusterNodeFlags(sds ci, uint16_t flags) { * See clusterGenNodesDescription() top comment for more information. * * The function returns the string representation as an SDS string. */ -sds clusterGenNodeDescription(clusterNode *node) { +sds clusterGenNodeDescription(clusterNode *node, int use_pport) { int j, start; sds ci; + int port = use_pport && node->pport ? node->pport : node->port; /* Node coordinates */ ci = sdscatlen(sdsempty(),node->name,CLUSTER_NAMELEN); ci = sdscatfmt(ci," %s:%i@%i ", node->ip, - node->port, + port, node->cport); /* Flags */ @@ -4250,10 +4286,13 @@ void clusterGenNodesSlotsInfo(int filter) { * include all the known nodes in the representation, including nodes in * the HANDSHAKE state. * + * Setting use_pport to 1 in a TLS cluster makes the result contain the + * plaintext client port rather then the TLS client port of each node. + * * The representation obtained using this function is used for the output * of the CLUSTER NODES function, and as format for the cluster * configuration file (nodes.conf) for a given node. */ -sds clusterGenNodesDescription(int filter) { +sds clusterGenNodesDescription(int filter, int use_pport) { sds ci = sdsempty(), ni; dictIterator *di; dictEntry *de; @@ -4266,7 +4305,7 @@ sds clusterGenNodesDescription(int filter) { clusterNode *node = dictGetVal(de); if (node->flags & filter) continue; - ni = clusterGenNodeDescription(node); + ni = clusterGenNodeDescription(node, use_pport); ci = sdscatsds(ci,ni); sdsfree(ni); ci = sdscatlen(ci,"\n",1); @@ -4313,7 +4352,37 @@ int getSlotOrReply(client *c, robj *o) { return (int) slot; } -void clusterReplyMultiBulkSlots(client *c) { +void addNodeReplyForClusterSlot(client *c, clusterNode *node, int start_slot, int end_slot) { + int i, nested_elements = 3; /* slots (2) + master addr (1) */ + void *nested_replylen = addReplyDeferredLen(c); + addReplyLongLong(c, start_slot); + addReplyLongLong(c, end_slot); + addReplyArrayLen(c, 3); + addReplyBulkCString(c, node->ip); + /* Report non-TLS ports to non-TLS client in TLS cluster if available. */ + int use_pport = (server.tls_cluster && + c->conn && connGetType(c->conn) != CONN_TYPE_TLS); + addReplyLongLong(c, use_pport && node->pport ? node->pport : node->port); + addReplyBulkCBuffer(c, node->name, CLUSTER_NAMELEN); + + /* Remaining nodes in reply are replicas for slot range */ + for (i = 0; i < node->numslaves; i++) { + /* This loop is copy/pasted from clusterGenNodeDescription() + * with modifications for per-slot node aggregation. */ + if (nodeFailed(node->slaves[i])) continue; + addReplyArrayLen(c, 3); + addReplyBulkCString(c, node->slaves[i]->ip); + /* Report slave's non-TLS port to non-TLS client in TLS cluster */ + addReplyLongLong(c, (use_pport && node->slaves[i]->pport ? + node->slaves[i]->pport : + node->slaves[i]->port)); + addReplyBulkCBuffer(c, node->slaves[i]->name, CLUSTER_NAMELEN); + nested_elements++; + } + setDeferredArrayLen(c, nested_replylen, nested_elements); +} + +void clusterReplyMultiBulkSlots(client * c) { /* Format: 1) 1) start slot * 2) end slot * 3) 1) master IP @@ -4324,69 +4393,29 @@ void clusterReplyMultiBulkSlots(client *c) { * 3) node ID * ... continued until done */ - - int num_masters = 0; + clusterNode *n = NULL; + int num_masters = 0, start = -1; void *slot_replylen = addReplyDeferredLen(c); - dictEntry *de; - dictIterator *di = dictGetSafeIterator(server.cluster->nodes); - while((de = dictNext(di)) != NULL) { - clusterNode *node = dictGetVal(de); - int j = 0, start = -1; - int i, nested_elements = 0; - - /* Skip slaves (that are iterated when producing the output of their - * master) and masters not serving any slot. */ - if (!nodeIsMaster(node) || node->numslots == 0) continue; - - for(i = 0; i < node->numslaves; i++) { - if (nodeFailed(node->slaves[i])) continue; - nested_elements++; + for (int i = 0; i <= CLUSTER_SLOTS; i++) { + /* Find start node and slot id. */ + if (n == NULL) { + if (i == CLUSTER_SLOTS) break; + n = server.cluster->slots[i]; + start = i; + continue; } - for (j = 0; j < CLUSTER_SLOTS; j++) { - int bit, i; - - if ((bit = clusterNodeGetSlotBit(node,j)) != 0) { - if (start == -1) start = j; - } - if (start != -1 && (!bit || j == CLUSTER_SLOTS-1)) { - addReplyArrayLen(c, nested_elements + 3); /* slots (2) + master addr (1). */ - - if (bit && j == CLUSTER_SLOTS-1) j++; - - /* If slot exists in output map, add to it's list. - * else, create a new output map for this slot */ - if (start == j-1) { - addReplyLongLong(c, start); /* only one slot; low==high */ - addReplyLongLong(c, start); - } else { - addReplyLongLong(c, start); /* low */ - addReplyLongLong(c, j-1); /* high */ - } - start = -1; - - /* First node reply position is always the master */ - addReplyArrayLen(c, 3); - addReplyBulkCString(c, node->ip); - addReplyLongLong(c, node->port); - addReplyBulkCBuffer(c, node->name, CLUSTER_NAMELEN); - - /* Remaining nodes in reply are replicas for slot range */ - for (i = 0; i < node->numslaves; i++) { - /* This loop is copy/pasted from clusterGenNodeDescription() - * with modifications for per-slot node aggregation */ - if (nodeFailed(node->slaves[i])) continue; - addReplyArrayLen(c, 3); - addReplyBulkCString(c, node->slaves[i]->ip); - addReplyLongLong(c, node->slaves[i]->port); - addReplyBulkCBuffer(c, node->slaves[i]->name, CLUSTER_NAMELEN); - } - num_masters++; - } + /* Add cluster slots info when occur different node with start + * or end of slot. */ + if (i == CLUSTER_SLOTS || n != server.cluster->slots[i]) { + addNodeReplyForClusterSlot(c, n, start, i-1); + num_masters++; + if (i == CLUSTER_SLOTS) break; + n = server.cluster->slots[i]; + start = i; } } - dictReleaseIterator(di); setDeferredArrayLen(c, slot_replylen, num_masters); } @@ -4475,7 +4504,11 @@ NULL } } else if (!strcasecmp(c->argv[1]->ptr,"nodes") && c->argc == 2) { /* CLUSTER NODES */ - sds nodes = clusterGenNodesDescription(0); + /* Report plaintext ports, only if cluster is TLS but client is known to + * be non-TLS). */ + int use_pport = (server.tls_cluster && + c->conn && connGetType(c->conn) != CONN_TYPE_TLS); + sds nodes = clusterGenNodesDescription(0, use_pport); addReplyVerbatim(c,nodes,sdslen(nodes),"txt"); sdsfree(nodes); } else if (!strcasecmp(c->argv[1]->ptr,"myid") && c->argc == 2) { @@ -4851,9 +4884,12 @@ NULL return; } + /* Use plaintext port if cluster is TLS but client is non-TLS. */ + int use_pport = (server.tls_cluster && + c->conn && connGetType(c->conn) != CONN_TYPE_TLS); addReplyArrayLen(c,n->numslaves); for (j = 0; j < n->numslaves; j++) { - sds ni = clusterGenNodeDescription(n->slaves[j]); + sds ni = clusterGenNodeDescription(n->slaves[j], use_pport); addReplyBulkCString(c,ni); sdsfree(ni); } @@ -5824,18 +5860,14 @@ clusterNode *getNodeByQuery(client *c, struct redisCommand *cmd, robj **argv, in * cluster is down. */ if (error_code) *error_code = CLUSTER_REDIR_DOWN_STATE; return NULL; - } else if ((cmd->flags & CMD_WRITE) && !(cmd->proc == evalCommand) - && !(cmd->proc == evalShaCommand)) - { - /* The cluster is configured to allow read only commands - * but this command is neither readonly, nor EVAL or - * EVALSHA. */ + } else if (cmd->flags & CMD_WRITE) { + /* The cluster is configured to allow read only commands */ if (error_code) *error_code = CLUSTER_REDIR_DOWN_RO_STATE; return NULL; } else { /* Fall through and allow the command to be executed: * this happens when server.cluster_allow_reads_when_down is - * true and the command is a readonly command or EVAL / EVALSHA. */ + * true and the command is not a write command */ } } @@ -5876,7 +5908,7 @@ clusterNode *getNodeByQuery(client *c, struct redisCommand *cmd, robj **argv, in int is_write_command = (c->cmd->flags & CMD_WRITE) || (c->cmd->proc == execCommand && (c->mstate.cmd_flags & CMD_WRITE)); if (c->flags & CLIENT_READONLY && - (!is_write_command || cmd->proc == evalCommand || cmd->proc == evalShaCommand) && + !is_write_command && nodeIsSlave(myself) && myself->slaveof == n) { @@ -5913,10 +5945,15 @@ void clusterRedirectClient(client *c, clusterNode *n, int hashslot, int error_co } else if (error_code == CLUSTER_REDIR_MOVED || error_code == CLUSTER_REDIR_ASK) { + /* Redirect to IP:port. Include plaintext port if cluster is TLS but + * client is non-TLS. */ + int use_pport = (server.tls_cluster && + c->conn && connGetType(c->conn) != CONN_TYPE_TLS); + int port = use_pport && n->pport ? n->pport : n->port; addReplyErrorSds(c,sdscatprintf(sdsempty(), "-%s %d %s:%d", (error_code == CLUSTER_REDIR_ASK) ? "ASK" : "MOVED", - hashslot,n->ip,n->port)); + hashslot, n->ip, port)); } else { serverPanic("getNodeByQuery() unknown error."); } diff --git a/src/cluster.h b/src/cluster.h index 716c0d49c..f476a50a0 100644 --- a/src/cluster.h +++ b/src/cluster.h @@ -135,7 +135,9 @@ typedef struct clusterNode { mstime_t orphaned_time; /* Starting time of orphaned master condition */ long long repl_offset; /* Last known repl offset for this node. */ char ip[NET_IP_STR_LEN]; /* Latest known IP address of this node */ - int port; /* Latest known clients port of this node */ + int port; /* Latest known clients port (TLS or plain). */ + int pport; /* Latest known clients plaintext port. Only used + if the main clients port is for TLS. */ int cport; /* Latest known cluster port of this node. */ clusterLink *link; /* TCP/IP link with this node */ list *fail_reports; /* List of nodes signaling this as failing */ @@ -168,7 +170,7 @@ typedef struct clusterState { clusterNode *mf_slave; /* Slave performing the manual failover. */ /* Manual failover state of slave. */ long long mf_master_offset; /* Master offset the slave needs to start MF - or zero if still not received. */ + or -1 if still not received. */ int mf_can_start; /* If non-zero signal that the manual failover can start requesting masters vote. */ /* The following fields are used by masters to take state on elections. */ @@ -194,7 +196,8 @@ typedef struct { uint16_t port; /* base port last time it was seen */ uint16_t cport; /* cluster port last time it was seen */ uint16_t flags; /* node->flags copy */ - uint32_t notused1; + uint16_t pport; /* plaintext-port, when base port is TLS */ + uint16_t notused1; } clusterMsgDataGossip; typedef struct { @@ -267,7 +270,8 @@ typedef struct { unsigned char myslots[CLUSTER_SLOTS/8]; char slaveof[CLUSTER_NAMELEN]; char myip[NET_IP_STR_LEN]; /* Sender IP, if not all zeroed. */ - char notused1[34]; /* 34 bytes reserved for future usage. */ + char notused1[32]; /* 32 bytes reserved for future usage. */ + uint16_t pport; /* Sender TCP plaintext port, if base port is TLS */ uint16_t cport; /* Sender TCP cluster bus port */ uint16_t flags; /* Sender node flags */ unsigned char state; /* Cluster state from the POV of the sender */ diff --git a/src/config.c b/src/config.c index 1d6ce6b8c..9861c5f52 100644 --- a/src/config.c +++ b/src/config.c @@ -229,8 +229,6 @@ typedef union typeData { typedef struct typeInterface { /* Called on server start, to init the server with default value */ void (*init)(typeData data); - /* Called on server start, should return 1 on success, 0 on error and should set err */ - int (*load)(typeData data, sds *argc, int argv, const char **err); /* Called on server startup and CONFIG SET, returns 1 on success, 0 on error * and can set a verbose err string, update is true when called from CONFIG SET */ int (*set)(typeData data, sds value, int update, const char **err); @@ -243,11 +241,16 @@ typedef struct typeInterface { typedef struct standardConfig { const char *name; /* The user visible name of this config */ const char *alias; /* An alias that can also be used for this config */ - const int modifiable; /* Can this value be updated by CONFIG SET? */ + const unsigned int flags; /* Flags for this specific config */ typeInterface interface; /* The function pointers that define the type interface */ typeData data; /* The type specific data exposed used by the interface */ } standardConfig; +#define MODIFIABLE_CONFIG 0 /* This is the implied default for a standard + * config, which is mutable. */ +#define IMMUTABLE_CONFIG (1ULL<<0) /* Can this value only be set at startup? */ +#define SENSITIVE_CONFIG (1ULL<<1) /* Does this value contain sensitive information */ + standardConfig configs[]; /*----------------------------------------------------------------------------- @@ -618,6 +621,10 @@ void loadServerConfigFromString(char *config) { goto loaderr; } + /* To ensure backward compatibility and work while hz is out of range */ + if (server.config_hz < CONFIG_MIN_HZ) server.config_hz = CONFIG_MIN_HZ; + if (server.config_hz > CONFIG_MAX_HZ) server.config_hz = CONFIG_MAX_HZ; + sdsfreesplitres(lines,totlines); return; @@ -714,9 +721,13 @@ void configSetCommand(client *c) { /* Iterate the configs that are standard */ for (standardConfig *config = configs; config->name != NULL; config++) { - if(config->modifiable && (!strcasecmp(c->argv[2]->ptr,config->name) || + if (!(config->flags & IMMUTABLE_CONFIG) && + (!strcasecmp(c->argv[2]->ptr,config->name) || (config->alias && !strcasecmp(c->argv[2]->ptr,config->alias)))) { + if (config->flags & SENSITIVE_CONFIG) { + preventCommandLogging(c); + } if (!config->interface.set(config->data,o->ptr,1,&errstr)) { goto badfmt; } @@ -1370,14 +1381,19 @@ void rewriteConfigSaveOption(struct rewriteConfigState *state) { return; } - /* Note that if there are no save parameters at all, all the current - * config line with "save" will be detected as orphaned and deleted, - * resulting into no RDB persistence as expected. */ - for (j = 0; j < server.saveparamslen; j++) { - line = sdscatprintf(sdsempty(),"save %ld %d", - (long) server.saveparams[j].seconds, server.saveparams[j].changes); - rewriteConfigRewriteLine(state,"save",line,1); + /* Rewrite save parameters, or an empty 'save ""' line to avoid the + * defaults from being used. + */ + if (!server.saveparamslen) { + rewriteConfigRewriteLine(state,"save",sdsnew("save \"\""),1); + } else { + for (j = 0; j < server.saveparamslen; j++) { + line = sdscatprintf(sdsempty(),"save %ld %d", + (long) server.saveparams[j].seconds, server.saveparams[j].changes); + rewriteConfigRewriteLine(state,"save",line,1); + } } + /* Mark "save" as processed in case server.saveparamslen is zero. */ rewriteConfigMarkAsProcessed(state,"save"); } @@ -1711,13 +1727,10 @@ int rewriteConfig(char *path, int force_all) { #define LOADBUF_SIZE 256 static char loadbuf[LOADBUF_SIZE]; -#define MODIFIABLE_CONFIG 1 -#define IMMUTABLE_CONFIG 0 - -#define embedCommonConfig(config_name, config_alias, is_modifiable) \ +#define embedCommonConfig(config_name, config_alias, config_flags) \ .name = (config_name), \ .alias = (config_alias), \ - .modifiable = (is_modifiable), + .flags = (config_flags), #define embedConfigInterface(initfn, setfn, getfn, rewritefn) .interface = { \ .init = (initfn), \ @@ -1768,8 +1781,8 @@ static void boolConfigRewrite(typeData data, const char *name, struct rewriteCon rewriteConfigYesNoOption(state, name,*(data.yesno.config), data.yesno.default_value); } -#define createBoolConfig(name, alias, modifiable, config_addr, default, is_valid, update) { \ - embedCommonConfig(name, alias, modifiable) \ +#define createBoolConfig(name, alias, flags, config_addr, default, is_valid, update) { \ + embedCommonConfig(name, alias, flags) \ embedConfigInterface(boolConfigInit, boolConfigSet, boolConfigGet, boolConfigRewrite) \ .data.yesno = { \ .config = &(config_addr), \ @@ -1841,8 +1854,8 @@ static void sdsConfigRewrite(typeData data, const char *name, struct rewriteConf #define ALLOW_EMPTY_STRING 0 #define EMPTY_STRING_IS_NULL 1 -#define createStringConfig(name, alias, modifiable, empty_to_null, config_addr, default, is_valid, update) { \ - embedCommonConfig(name, alias, modifiable) \ +#define createStringConfig(name, alias, flags, empty_to_null, config_addr, default, is_valid, update) { \ + embedCommonConfig(name, alias, flags) \ embedConfigInterface(stringConfigInit, stringConfigSet, stringConfigGet, stringConfigRewrite) \ .data.string = { \ .config = &(config_addr), \ @@ -1853,8 +1866,8 @@ static void sdsConfigRewrite(typeData data, const char *name, struct rewriteConf } \ } -#define createSDSConfig(name, alias, modifiable, empty_to_null, config_addr, default, is_valid, update) { \ - embedCommonConfig(name, alias, modifiable) \ +#define createSDSConfig(name, alias, flags, empty_to_null, config_addr, default, is_valid, update) { \ + embedCommonConfig(name, alias, flags) \ embedConfigInterface(sdsConfigInit, sdsConfigSet, sdsConfigGet, sdsConfigRewrite) \ .data.sds = { \ .config = &(config_addr), \ @@ -1909,8 +1922,8 @@ static void enumConfigRewrite(typeData data, const char *name, struct rewriteCon rewriteConfigEnumOption(state, name,*(data.enumd.config), data.enumd.enum_value, data.enumd.default_value); } -#define createEnumConfig(name, alias, modifiable, enum, config_addr, default, is_valid, update) { \ - embedCommonConfig(name, alias, modifiable) \ +#define createEnumConfig(name, alias, flags, enum, config_addr, default, is_valid, update) { \ + embedCommonConfig(name, alias, flags) \ embedConfigInterface(enumConfigInit, enumConfigSet, enumConfigGet, enumConfigRewrite) \ .data.enumd = { \ .config = &(config_addr), \ @@ -2063,8 +2076,8 @@ static void numericConfigRewrite(typeData data, const char *name, struct rewrite #define INTEGER_CONFIG 0 #define MEMORY_CONFIG 1 -#define embedCommonNumericalConfig(name, alias, modifiable, lower, upper, config_addr, default, memory, is_valid, update) { \ - embedCommonConfig(name, alias, modifiable) \ +#define embedCommonNumericalConfig(name, alias, flags, lower, upper, config_addr, default, memory, is_valid, update) { \ + embedCommonConfig(name, alias, flags) \ embedConfigInterface(numericConfigInit, numericConfigSet, numericConfigGet, numericConfigRewrite) \ .data.numeric = { \ .lower_bound = (lower), \ @@ -2074,71 +2087,71 @@ static void numericConfigRewrite(typeData data, const char *name, struct rewrite .update_fn = (update), \ .is_memory = (memory), -#define createIntConfig(name, alias, modifiable, lower, upper, config_addr, default, memory, is_valid, update) \ - embedCommonNumericalConfig(name, alias, modifiable, lower, upper, config_addr, default, memory, is_valid, update) \ +#define createIntConfig(name, alias, flags, lower, upper, config_addr, default, memory, is_valid, update) \ + embedCommonNumericalConfig(name, alias, flags, lower, upper, config_addr, default, memory, is_valid, update) \ .numeric_type = NUMERIC_TYPE_INT, \ .config.i = &(config_addr) \ } \ } -#define createUIntConfig(name, alias, modifiable, lower, upper, config_addr, default, memory, is_valid, update) \ - embedCommonNumericalConfig(name, alias, modifiable, lower, upper, config_addr, default, memory, is_valid, update) \ +#define createUIntConfig(name, alias, flags, lower, upper, config_addr, default, memory, is_valid, update) \ + embedCommonNumericalConfig(name, alias, flags, lower, upper, config_addr, default, memory, is_valid, update) \ .numeric_type = NUMERIC_TYPE_UINT, \ .config.ui = &(config_addr) \ } \ } -#define createLongConfig(name, alias, modifiable, lower, upper, config_addr, default, memory, is_valid, update) \ - embedCommonNumericalConfig(name, alias, modifiable, lower, upper, config_addr, default, memory, is_valid, update) \ +#define createLongConfig(name, alias, flags, lower, upper, config_addr, default, memory, is_valid, update) \ + embedCommonNumericalConfig(name, alias, flags, lower, upper, config_addr, default, memory, is_valid, update) \ .numeric_type = NUMERIC_TYPE_LONG, \ .config.l = &(config_addr) \ } \ } -#define createULongConfig(name, alias, modifiable, lower, upper, config_addr, default, memory, is_valid, update) \ - embedCommonNumericalConfig(name, alias, modifiable, lower, upper, config_addr, default, memory, is_valid, update) \ +#define createULongConfig(name, alias, flags, lower, upper, config_addr, default, memory, is_valid, update) \ + embedCommonNumericalConfig(name, alias, flags, lower, upper, config_addr, default, memory, is_valid, update) \ .numeric_type = NUMERIC_TYPE_ULONG, \ .config.ul = &(config_addr) \ } \ } -#define createLongLongConfig(name, alias, modifiable, lower, upper, config_addr, default, memory, is_valid, update) \ - embedCommonNumericalConfig(name, alias, modifiable, lower, upper, config_addr, default, memory, is_valid, update) \ +#define createLongLongConfig(name, alias, flags, lower, upper, config_addr, default, memory, is_valid, update) \ + embedCommonNumericalConfig(name, alias, flags, lower, upper, config_addr, default, memory, is_valid, update) \ .numeric_type = NUMERIC_TYPE_LONG_LONG, \ .config.ll = &(config_addr) \ } \ } -#define createULongLongConfig(name, alias, modifiable, lower, upper, config_addr, default, memory, is_valid, update) \ - embedCommonNumericalConfig(name, alias, modifiable, lower, upper, config_addr, default, memory, is_valid, update) \ +#define createULongLongConfig(name, alias, flags, lower, upper, config_addr, default, memory, is_valid, update) \ + embedCommonNumericalConfig(name, alias, flags, lower, upper, config_addr, default, memory, is_valid, update) \ .numeric_type = NUMERIC_TYPE_ULONG_LONG, \ .config.ull = &(config_addr) \ } \ } -#define createSizeTConfig(name, alias, modifiable, lower, upper, config_addr, default, memory, is_valid, update) \ - embedCommonNumericalConfig(name, alias, modifiable, lower, upper, config_addr, default, memory, is_valid, update) \ +#define createSizeTConfig(name, alias, flags, lower, upper, config_addr, default, memory, is_valid, update) \ + embedCommonNumericalConfig(name, alias, flags, lower, upper, config_addr, default, memory, is_valid, update) \ .numeric_type = NUMERIC_TYPE_SIZE_T, \ .config.st = &(config_addr) \ } \ } -#define createSSizeTConfig(name, alias, modifiable, lower, upper, config_addr, default, memory, is_valid, update) \ - embedCommonNumericalConfig(name, alias, modifiable, lower, upper, config_addr, default, memory, is_valid, update) \ +#define createSSizeTConfig(name, alias, flags, lower, upper, config_addr, default, memory, is_valid, update) \ + embedCommonNumericalConfig(name, alias, flags, lower, upper, config_addr, default, memory, is_valid, update) \ .numeric_type = NUMERIC_TYPE_SSIZE_T, \ .config.sst = &(config_addr) \ } \ } -#define createTimeTConfig(name, alias, modifiable, lower, upper, config_addr, default, memory, is_valid, update) \ - embedCommonNumericalConfig(name, alias, modifiable, lower, upper, config_addr, default, memory, is_valid, update) \ +#define createTimeTConfig(name, alias, flags, lower, upper, config_addr, default, memory, is_valid, update) \ + embedCommonNumericalConfig(name, alias, flags, lower, upper, config_addr, default, memory, is_valid, update) \ .numeric_type = NUMERIC_TYPE_TIME_T, \ .config.tt = &(config_addr) \ } \ } -#define createOffTConfig(name, alias, modifiable, lower, upper, config_addr, default, memory, is_valid, update) \ - embedCommonNumericalConfig(name, alias, modifiable, lower, upper, config_addr, default, memory, is_valid, update) \ +#define createOffTConfig(name, alias, flags, lower, upper, config_addr, default, memory, is_valid, update) \ + embedCommonNumericalConfig(name, alias, flags, lower, upper, config_addr, default, memory, is_valid, update) \ .numeric_type = NUMERIC_TYPE_OFF_T, \ .config.ot = &(config_addr) \ } \ @@ -2425,13 +2438,15 @@ standardConfig configs[] = { createBoolConfig("crash-memcheck-enabled", NULL, MODIFIABLE_CONFIG, server.memcheck_enabled, 1, NULL, NULL), createBoolConfig("use-exit-on-panic", NULL, MODIFIABLE_CONFIG, server.use_exit_on_panic, 0, NULL, NULL), createBoolConfig("disable-thp", NULL, MODIFIABLE_CONFIG, server.disable_thp, 1, NULL, NULL), + createBoolConfig("cluster-allow-replica-migration", NULL, MODIFIABLE_CONFIG, server.cluster_allow_replica_migration, 1, NULL, NULL), + createBoolConfig("replica-announced", NULL, MODIFIABLE_CONFIG, server.replica_announced, 1, NULL, NULL), /* String Configs */ createStringConfig("aclfile", NULL, IMMUTABLE_CONFIG, ALLOW_EMPTY_STRING, server.acl_filename, "", NULL, NULL), createStringConfig("unixsocket", NULL, IMMUTABLE_CONFIG, EMPTY_STRING_IS_NULL, server.unixsocket, NULL, NULL, NULL), createStringConfig("pidfile", NULL, IMMUTABLE_CONFIG, EMPTY_STRING_IS_NULL, server.pidfile, NULL, NULL, NULL), createStringConfig("replica-announce-ip", "slave-announce-ip", MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.slave_announce_ip, NULL, NULL, NULL), - createStringConfig("masteruser", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.masteruser, NULL, NULL, NULL), + createStringConfig("masteruser", NULL, MODIFIABLE_CONFIG | SENSITIVE_CONFIG, EMPTY_STRING_IS_NULL, server.masteruser, NULL, NULL, NULL), createStringConfig("cluster-announce-ip", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.cluster_announce_ip, NULL, NULL, NULL), createStringConfig("syslog-ident", NULL, IMMUTABLE_CONFIG, ALLOW_EMPTY_STRING, server.syslog_ident, "redis", NULL, NULL), createStringConfig("dbfilename", NULL, MODIFIABLE_CONFIG, ALLOW_EMPTY_STRING, server.rdb_filename, "dump.rdb", isValidDBfilename, NULL), @@ -2444,8 +2459,8 @@ standardConfig configs[] = { createStringConfig("proc-title-template", NULL, MODIFIABLE_CONFIG, ALLOW_EMPTY_STRING, server.proc_title_template, CONFIG_DEFAULT_PROC_TITLE_TEMPLATE, isValidProcTitleTemplate, updateProcTitleTemplate), /* SDS Configs */ - createSDSConfig("masterauth", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.masterauth, NULL, NULL, NULL), - createSDSConfig("requirepass", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.requirepass, NULL, NULL, updateRequirePass), + createSDSConfig("masterauth", NULL, MODIFIABLE_CONFIG | SENSITIVE_CONFIG, EMPTY_STRING_IS_NULL, server.masterauth, NULL, NULL, NULL), + createSDSConfig("requirepass", NULL, MODIFIABLE_CONFIG | SENSITIVE_CONFIG, EMPTY_STRING_IS_NULL, server.requirepass, NULL, NULL, updateRequirePass), /* Enum Configs */ createEnumConfig("supervised", NULL, IMMUTABLE_CONFIG, supervised_mode_enum, server.supervised_mode, SUPERVISED_NONE, NULL, NULL), @@ -2455,7 +2470,7 @@ standardConfig configs[] = { createEnumConfig("maxmemory-policy", NULL, MODIFIABLE_CONFIG, maxmemory_policy_enum, server.maxmemory_policy, MAXMEMORY_NO_EVICTION, NULL, NULL), createEnumConfig("appendfsync", NULL, MODIFIABLE_CONFIG, aof_fsync_enum, server.aof_fsync, AOF_FSYNC_EVERYSEC, NULL, NULL), createEnumConfig("oom-score-adj", NULL, MODIFIABLE_CONFIG, oom_score_adj_enum, server.oom_score_adj, OOM_SCORE_ADJ_NO, NULL, updateOOMScoreAdj), - createEnumConfig("acl-pubsub-default", NULL, MODIFIABLE_CONFIG, acl_pubsub_default_enum, server.acl_pubusub_default, USER_FLAG_ALLCHANNELS, NULL, NULL), + createEnumConfig("acl-pubsub-default", NULL, MODIFIABLE_CONFIG, acl_pubsub_default_enum, server.acl_pubsub_default, USER_FLAG_ALLCHANNELS, NULL, NULL), createEnumConfig("sanitize-dump-payload", NULL, MODIFIABLE_CONFIG, sanitize_dump_payload_enum, server.sanitize_dump_payload, SANITIZE_DUMP_NO, NULL, NULL), /* Integer configs */ @@ -2482,6 +2497,7 @@ standardConfig configs[] = { createIntConfig("tcp-backlog", NULL, IMMUTABLE_CONFIG, 0, INT_MAX, server.tcp_backlog, 511, INTEGER_CONFIG, NULL, NULL), /* TCP listen backlog. */ createIntConfig("cluster-announce-bus-port", NULL, MODIFIABLE_CONFIG, 0, 65535, server.cluster_announce_bus_port, 0, INTEGER_CONFIG, NULL, NULL), /* Default: Use +10000 offset. */ createIntConfig("cluster-announce-port", NULL, MODIFIABLE_CONFIG, 0, 65535, server.cluster_announce_port, 0, INTEGER_CONFIG, NULL, NULL), /* Use server.port */ + createIntConfig("cluster-announce-tls-port", NULL, MODIFIABLE_CONFIG, 0, 65535, server.cluster_announce_tls_port, 0, INTEGER_CONFIG, NULL, NULL), /* Use server.tls_port */ createIntConfig("repl-timeout", NULL, MODIFIABLE_CONFIG, 1, INT_MAX, server.repl_timeout, 60, INTEGER_CONFIG, NULL, NULL), createIntConfig("repl-ping-replica-period", "repl-ping-slave-period", MODIFIABLE_CONFIG, 1, INT_MAX, server.repl_ping_slave_period, 10, INTEGER_CONFIG, NULL, NULL), createIntConfig("list-compress-depth", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.list_compress_depth, 0, INTEGER_CONFIG, NULL, NULL), @@ -2539,8 +2555,10 @@ standardConfig configs[] = { createBoolConfig("tls-session-caching", NULL, MODIFIABLE_CONFIG, server.tls_ctx_config.session_caching, 1, NULL, updateTlsCfgBool), createStringConfig("tls-cert-file", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.tls_ctx_config.cert_file, NULL, NULL, updateTlsCfg), createStringConfig("tls-key-file", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.tls_ctx_config.key_file, NULL, NULL, updateTlsCfg), + createStringConfig("tls-key-file-pass", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.tls_ctx_config.key_file_pass, NULL, NULL, updateTlsCfg), createStringConfig("tls-client-cert-file", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.tls_ctx_config.client_cert_file, NULL, NULL, updateTlsCfg), createStringConfig("tls-client-key-file", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.tls_ctx_config.client_key_file, NULL, NULL, updateTlsCfg), + createStringConfig("tls-client-key-file-pass", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.tls_ctx_config.client_key_file_pass, NULL, NULL, updateTlsCfg), createStringConfig("tls-dh-params-file", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.tls_ctx_config.dh_params_file, NULL, NULL, updateTlsCfg), createStringConfig("tls-ca-cert-file", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.tls_ctx_config.ca_cert_file, NULL, NULL, updateTlsCfg), createStringConfig("tls-ca-cert-dir", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.tls_ctx_config.ca_cert_dir, NULL, NULL, updateTlsCfg), diff --git a/src/crc64.c b/src/crc64.c index 6c9432c4a..d4db4158e 100644 --- a/src/crc64.c +++ b/src/crc64.c @@ -127,9 +127,10 @@ uint64_t crc64(uint64_t crc, const unsigned char *s, uint64_t l) { #include #define UNUSED(x) (void)(x) -int crc64Test(int argc, char *argv[]) { +int crc64Test(int argc, char *argv[], int accurate) { UNUSED(argc); UNUSED(argv); + UNUSED(accurate); crc64_init(); printf("[calcula]: e9c6d914c4b8d9ca == %016" PRIx64 "\n", (uint64_t)_crc64(0, "123456789", 9)); diff --git a/src/crc64.h b/src/crc64.h index 60c42345f..38b0b6387 100644 --- a/src/crc64.h +++ b/src/crc64.h @@ -7,7 +7,7 @@ void crc64_init(void); uint64_t crc64(uint64_t crc, const unsigned char *s, uint64_t l); #ifdef REDIS_TEST -int crc64Test(int argc, char *argv[]); +int crc64Test(int argc, char *argv[], int accurate); #endif #endif diff --git a/src/db.c b/src/db.c index 57705f003..ec68c228c 100644 --- a/src/db.c +++ b/src/db.c @@ -139,9 +139,9 @@ robj *lookupKeyReadWithFlags(redisDb *db, robj *key, int flags) { keymiss: if (!(flags & LOOKUP_NONOTIFY)) { - server.stat_keyspace_misses++; notifyKeyspaceEvent(NOTIFY_KEY_MISS, "keymiss", key, db->id); } + server.stat_keyspace_misses++; return NULL; } @@ -164,16 +164,24 @@ robj *lookupKeyWriteWithFlags(redisDb *db, robj *key, int flags) { robj *lookupKeyWrite(redisDb *db, robj *key) { return lookupKeyWriteWithFlags(db, key, LOOKUP_NONE); } - +static void SentReplyOnKeyMiss(client *c, robj *reply){ + serverAssert(sdsEncodedObject(reply)); + sds rep = reply->ptr; + if (sdslen(rep) > 1 && rep[0] == '-'){ + addReplyErrorObject(c, reply); + } else { + addReply(c,reply); + } +} robj *lookupKeyReadOrReply(client *c, robj *key, robj *reply) { robj *o = lookupKeyRead(c->db, key); - if (!o) addReply(c,reply); + if (!o) SentReplyOnKeyMiss(c, reply); return o; } robj *lookupKeyWriteOrReply(client *c, robj *key, robj *reply) { robj *o = lookupKeyWrite(c->db, key); - if (!o) addReply(c,reply); + if (!o) SentReplyOnKeyMiss(c, reply); return o; } @@ -1445,7 +1453,12 @@ void propagateExpire(redisDb *db, robj *key, int lazy) { incrRefCount(argv[0]); incrRefCount(argv[1]); + /* If the master decided to expire a key we must propagate it to replicas no matter what.. + * Even if module executed a command without asking for propagation. */ + int prev_replication_allowed = server.replication_allowed; + server.replication_allowed = 1; propagate(server.delCommand,db->id,argv,2,PROPAGATE_AOF|PROPAGATE_REPL); + server.replication_allowed = prev_replication_allowed; decrRefCount(argv[0]); decrRefCount(argv[1]); diff --git a/src/debug.c b/src/debug.c index e7fec293a..098ce6ef7 100644 --- a/src/debug.c +++ b/src/debug.c @@ -249,7 +249,7 @@ void xorObjectDigest(redisDb *db, robj *keyobj, unsigned char *digest, robj *o) } streamIteratorStop(&si); } else if (o->type == OBJ_MODULE) { - RedisModuleDigest md; + RedisModuleDigest md = {{0},{0}}; moduleValue *mv = o->ptr; moduleType *mt = mv->type; moduleInitDigestContext(md); @@ -441,7 +441,7 @@ void debugCommand(client *c) { " conflicting keys will generate an exception and kill the server." " * NOSAVE: the database will be loaded from an existing RDB file.", " Examples:", -" * DEBUG RELOAD: verify that the server is able to persist, flsuh and reload", +" * DEBUG RELOAD: verify that the server is able to persist, flush and reload", " the database.", " * DEBUG RELOAD NOSAVE: replace the current database with the contents of an", " existing RDB file.", @@ -473,7 +473,7 @@ NULL } else if (!strcasecmp(c->argv[1]->ptr,"segfault")) { *((char*)-1) = 'x'; } else if (!strcasecmp(c->argv[1]->ptr,"panic")) { - serverPanic("DEBUG PANIC called at Unix time %ld", time(NULL)); + serverPanic("DEBUG PANIC called at Unix time %lld", (long long)time(NULL)); } else if (!strcasecmp(c->argv[1]->ptr,"restart") || !strcasecmp(c->argv[1]->ptr,"crash-and-recover")) { @@ -1809,7 +1809,7 @@ void sigsegvHandler(int sig, siginfo_t *info, void *secret) { serverLog(LL_WARNING, "Accessing address: %p", (void*)info->si_addr); } - if (info->si_pid != -1) { + if (info->si_code <= SI_USER && info->si_pid != -1) { serverLog(LL_WARNING, "Killed by PID: %ld, UID: %d", (long) info->si_pid, info->si_uid); } diff --git a/src/defrag.c b/src/defrag.c index 2b28d523d..3dfb8679c 100644 --- a/src/defrag.c +++ b/src/defrag.c @@ -347,7 +347,9 @@ long activeDefragSdsListAndDict(list *l, dict *d, int dict_val_type) { if ((newsds = activeDefragSds(sdsele))) { /* When defragging an sds value, we need to update the dict key */ uint64_t hash = dictGetHash(d, newsds); - replaceSatelliteDictKeyPtrAndOrDefragDictEntry(d, sdsele, newsds, hash, &defragged); + dictEntry **deref = dictFindEntryRefByPtrAndHash(d, sdsele, hash); + if (deref) + (*deref)->key = newsds; ln->value = newsds; defragged++; } diff --git a/src/dict.c b/src/dict.c index 4a9f3fb0a..21c616e6f 100644 --- a/src/dict.c +++ b/src/dict.c @@ -45,11 +45,7 @@ #include "dict.h" #include "zmalloc.h" -#ifndef DICT_BENCHMARK_MAIN #include "redisassert.h" -#else -#include -#endif /* Using dictEnableResize() / dictDisableResize() we make possible to * enable/disable resizing of the hash table as needed. This is very important @@ -1175,20 +1171,18 @@ void dictGetStats(char *buf, size_t bufsize, dict *d) { /* ------------------------------- Benchmark ---------------------------------*/ -#ifdef DICT_BENCHMARK_MAIN - -#include "sds.h" +#ifdef REDIS_TEST uint64_t hashCallback(const void *key) { - return dictGenHashFunction((unsigned char*)key, sdslen((char*)key)); + return dictGenHashFunction((unsigned char*)key, strlen((char*)key)); } int compareCallback(void *privdata, const void *key1, const void *key2) { int l1,l2; DICT_NOTUSED(privdata); - l1 = sdslen((sds)key1); - l2 = sdslen((sds)key2); + l1 = strlen((char*)key1); + l2 = strlen((char*)key2); if (l1 != l2) return 0; return memcmp(key1, key2, l1) == 0; } @@ -1196,7 +1190,19 @@ int compareCallback(void *privdata, const void *key1, const void *key2) { void freeCallback(void *privdata, void *val) { DICT_NOTUSED(privdata); - sdsfree(val); + zfree(val); +} + +char *stringFromLongLong(long long value) { + char buf[32]; + int len; + char *s; + + len = sprintf(buf,"%lld",value); + s = zmalloc(len+1); + memcpy(s, buf, len); + s[len] = '\0'; + return s; } dictType BenchmarkDictType = { @@ -1215,22 +1221,26 @@ dictType BenchmarkDictType = { printf(msg ": %ld items in %lld ms\n", count, elapsed); \ } while(0) -/* dict-benchmark [count] */ -int main(int argc, char **argv) { +/* ./redis-server test dict [ | --accurate] */ +int dictTest(int argc, char **argv, int accurate) { long j; long long start, elapsed; dict *dict = dictCreate(&BenchmarkDictType,NULL); long count = 0; - if (argc == 2) { - count = strtol(argv[1],NULL,10); + if (argc == 4) { + if (accurate) { + count = 5000000; + } else { + count = strtol(argv[3],NULL,10); + } } else { - count = 5000000; + count = 5000; } start_benchmark(); for (j = 0; j < count; j++) { - int retval = dictAdd(dict,sdsfromlonglong(j),(void*)j); + int retval = dictAdd(dict,stringFromLongLong(j),(void*)j); assert(retval == DICT_OK); } end_benchmark("Inserting"); @@ -1243,28 +1253,28 @@ int main(int argc, char **argv) { start_benchmark(); for (j = 0; j < count; j++) { - sds key = sdsfromlonglong(j); + char *key = stringFromLongLong(j); dictEntry *de = dictFind(dict,key); assert(de != NULL); - sdsfree(key); + zfree(key); } end_benchmark("Linear access of existing elements"); start_benchmark(); for (j = 0; j < count; j++) { - sds key = sdsfromlonglong(j); + char *key = stringFromLongLong(j); dictEntry *de = dictFind(dict,key); assert(de != NULL); - sdsfree(key); + zfree(key); } end_benchmark("Linear access of existing elements (2nd round)"); start_benchmark(); for (j = 0; j < count; j++) { - sds key = sdsfromlonglong(rand() % count); + char *key = stringFromLongLong(rand() % count); dictEntry *de = dictFind(dict,key); assert(de != NULL); - sdsfree(key); + zfree(key); } end_benchmark("Random access of existing elements"); @@ -1277,17 +1287,17 @@ int main(int argc, char **argv) { start_benchmark(); for (j = 0; j < count; j++) { - sds key = sdsfromlonglong(rand() % count); + char *key = stringFromLongLong(rand() % count); key[0] = 'X'; dictEntry *de = dictFind(dict,key); assert(de == NULL); - sdsfree(key); + zfree(key); } end_benchmark("Accessing missing"); start_benchmark(); for (j = 0; j < count; j++) { - sds key = sdsfromlonglong(j); + char *key = stringFromLongLong(j); int retval = dictDelete(dict,key); assert(retval == DICT_OK); key[0] += 17; /* Change first number to letter. */ @@ -1295,5 +1305,7 @@ int main(int argc, char **argv) { assert(retval == DICT_OK); } end_benchmark("Removing and adding"); + dictRelease(dict); + return 0; } #endif diff --git a/src/dict.h b/src/dict.h index bd57f859e..7e2258960 100644 --- a/src/dict.h +++ b/src/dict.h @@ -201,4 +201,8 @@ extern dictType dictTypeHeapStringCopyKey; extern dictType dictTypeHeapStrings; extern dictType dictTypeHeapStringCopyKeyValue; +#ifdef REDIS_TEST +int dictTest(int argc, char *argv[], int accurate); +#endif + #endif /* __DICT_H */ diff --git a/src/endianconv.c b/src/endianconv.c index 918844e25..98ed405a5 100644 --- a/src/endianconv.c +++ b/src/endianconv.c @@ -105,11 +105,12 @@ uint64_t intrev64(uint64_t v) { #include #define UNUSED(x) (void)(x) -int endianconvTest(int argc, char *argv[]) { +int endianconvTest(int argc, char *argv[], int accurate) { char buf[32]; UNUSED(argc); UNUSED(argv); + UNUSED(accurate); sprintf(buf,"ciaoroma"); memrev16(buf); diff --git a/src/endianconv.h b/src/endianconv.h index 475f72b08..004749786 100644 --- a/src/endianconv.h +++ b/src/endianconv.h @@ -72,7 +72,7 @@ uint64_t intrev64(uint64_t v); #endif #ifdef REDIS_TEST -int endianconvTest(int argc, char *argv[]); +int endianconvTest(int argc, char *argv[], int accurate); #endif #endif diff --git a/src/help.h b/src/help.h index 97de00231..c14b53aba 100644 --- a/src/help.h +++ b/src/help.h @@ -184,7 +184,7 @@ struct commandHelp { 8, "6.2.0" }, { "CLIENT KILL", - "[ip:port] [ID client-id] [TYPE normal|master|slave|pubsub] [USER username] [ADDR ip:port] [SKIPME yes/no]", + "[ip:port] [ID client-id] [TYPE normal|master|slave|pubsub] [USER username] [ADDR ip:port] [LADDR ip:port] [SKIPME yes/no]", "Kill the connection of a client", 8, "2.4.0" }, diff --git a/src/intset.c b/src/intset.c index 74de87acb..1a64ecae8 100644 --- a/src/intset.c +++ b/src/intset.c @@ -284,7 +284,7 @@ size_t intsetBlobLen(intset *is) { return sizeof(intset)+intrev32ifbe(is->length)*intrev32ifbe(is->encoding); } -/* Validate the integrity of the data stracture. +/* Validate the integrity of the data structure. * when `deep` is 0, only the integrity of the header is validated. * when `deep` is 1, we make sure there are no duplicate or out of order records. */ int intsetValidateIntegrity(const unsigned char *p, size_t size, int deep) { @@ -392,7 +392,7 @@ static void checkConsistency(intset *is) { } #define UNUSED(x) (void)(x) -int intsetTest(int argc, char **argv) { +int intsetTest(int argc, char **argv, int accurate) { uint8_t success; int i; intset *is; @@ -400,6 +400,7 @@ int intsetTest(int argc, char **argv) { UNUSED(argc); UNUSED(argv); + UNUSED(accurate); printf("Value encodings: "); { assert(_intsetValueEncoding(-32768) == INTSET_ENC_INT16); @@ -424,6 +425,7 @@ int intsetTest(int argc, char **argv) { is = intsetAdd(is,4,&success); assert(success); is = intsetAdd(is,4,&success); assert(!success); ok(); + zfree(is); } printf("Large number of random adds: "); { @@ -436,6 +438,7 @@ int intsetTest(int argc, char **argv) { assert(intrev32ifbe(is->length) == inserts); checkConsistency(is); ok(); + zfree(is); } printf("Upgrade from int16 to int32: "); { @@ -447,6 +450,7 @@ int intsetTest(int argc, char **argv) { assert(intsetFind(is,32)); assert(intsetFind(is,65535)); checkConsistency(is); + zfree(is); is = intsetNew(); is = intsetAdd(is,32,NULL); @@ -457,6 +461,7 @@ int intsetTest(int argc, char **argv) { assert(intsetFind(is,-65535)); checkConsistency(is); ok(); + zfree(is); } printf("Upgrade from int16 to int64: "); { @@ -468,6 +473,7 @@ int intsetTest(int argc, char **argv) { assert(intsetFind(is,32)); assert(intsetFind(is,4294967295)); checkConsistency(is); + zfree(is); is = intsetNew(); is = intsetAdd(is,32,NULL); @@ -478,6 +484,7 @@ int intsetTest(int argc, char **argv) { assert(intsetFind(is,-4294967295)); checkConsistency(is); ok(); + zfree(is); } printf("Upgrade from int32 to int64: "); { @@ -489,6 +496,7 @@ int intsetTest(int argc, char **argv) { assert(intsetFind(is,65535)); assert(intsetFind(is,4294967295)); checkConsistency(is); + zfree(is); is = intsetNew(); is = intsetAdd(is,65535,NULL); @@ -499,6 +507,7 @@ int intsetTest(int argc, char **argv) { assert(intsetFind(is,-4294967295)); checkConsistency(is); ok(); + zfree(is); } printf("Stress lookups: "); { @@ -512,6 +521,7 @@ int intsetTest(int argc, char **argv) { for (i = 0; i < num; i++) intsetSearch(is,rand() % ((1<.\" in order to enable it. If we weren't in a deep space mission I'd suggest to take a look at http://redis.io/topics/latency-monitor.\n"); + report = sdscat(report,"I'm sorry, Dave, I can't do that. Latency monitoring is disabled in this Redis instance. You may use \"CONFIG SET latency-monitor-threshold .\" in order to enable it. If we weren't in a deep space mission I'd suggest to take a look at https://redis.io/topics/latency-monitor.\n"); return report; } @@ -426,7 +426,7 @@ sds createLatencyReport(void) { } if (advise_slowlog_inspect) { - report = sdscat(report,"- Check your Slow Log to understand what are the commands you are running which are too slow to execute. Please check http://redis.io/commands/slowlog for more information.\n"); + report = sdscat(report,"- Check your Slow Log to understand what are the commands you are running which are too slow to execute. Please check https://redis.io/commands/slowlog for more information.\n"); } /* Intrinsic latency. */ diff --git a/src/listpack.c b/src/listpack.c index a2255f0d7..ee256bad3 100644 --- a/src/listpack.c +++ b/src/listpack.c @@ -908,7 +908,7 @@ int lpValidateNext(unsigned char *lp, unsigned char **pp, size_t lpbytes) { #undef OUT_OF_RANGE } -/* Validate the integrity of the data stracture. +/* Validate the integrity of the data structure. * when `deep` is 0, only the integrity of the header is validated. * when `deep` is 1, we scan all the entries one by one. */ int lpValidateIntegrity(unsigned char *lp, size_t size, int deep){ diff --git a/src/module.c b/src/module.c index 274210590..05bf3a275 100644 --- a/src/module.c +++ b/src/module.c @@ -27,6 +27,30 @@ * POSSIBILITY OF SUCH DAMAGE. */ +/* -------------------------------------------------------------------------- + * Modules API documentation information + * + * The comments in this file are used to generate the API documentation on the + * Redis website. + * + * Each function starting with RM_ and preceded by a block comment is included + * in the API documentation. To hide an RM_ function, put a blank line between + * the comment and the function definition or put the comment inside the + * function body. + * + * The functions are divided into sections. Each section is preceded by a + * documentation block, which is comment block starting with a markdown level 2 + * heading, i.e. a line starting with ##, on the first line of the comment block + * (with the exception of a ----- line which can appear first). Other comment + * blocks, which are not intended for the modules API user, such as this comment + * block, do NOT start with a markdown level 2 heading, so they are included in + * the generated a API documentation. + * + * The documentation comments may contain markdown formatting. Some automatic + * replacements are done, such as the replacement of RM with RedisModule in + * function names. For details, see the script src/modules/gendoc.rb. + * -------------------------------------------------------------------------- */ + #include "server.h" #include "cluster.h" #include "slowlog.h" @@ -169,6 +193,7 @@ typedef struct RedisModuleCtx RedisModuleCtx; #define REDISMODULE_CTX_THREAD_SAFE (1<<4) #define REDISMODULE_CTX_BLOCKED_DISCONNECTED (1<<5) #define REDISMODULE_CTX_MODULE_COMMAND_CALL (1<<6) +#define REDISMODULE_CTX_MULTI_EMITTED (1<<7) /* This represents a Redis key opened with RM_OpenKey(). */ struct RedisModuleKey { @@ -396,7 +421,10 @@ void RM_FreeDict(RedisModuleCtx *ctx, RedisModuleDict *d); void RM_FreeServerInfo(RedisModuleCtx *ctx, RedisModuleServerInfoData *data); /* -------------------------------------------------------------------------- - * Heap allocation raw functions + * ## Heap allocation raw functions + * + * Memory allocated with these functions are taken into account by Redis key + * eviction algorithms and are reported in Redis memory usage information. * -------------------------------------------------------------------------- */ /* Use like malloc(). Memory allocated with this function is reported in @@ -579,13 +607,13 @@ int moduleDelKeyIfEmpty(RedisModuleKey *key) { * defined in the main executable having the same names. * -------------------------------------------------------------------------- */ -/* Lookup the requested module API and store the function pointer into the - * target pointer. The function returns REDISMODULE_ERR if there is no such - * named API, otherwise REDISMODULE_OK. - * - * This function is not meant to be used by modules developer, it is only - * used implicitly by including redismodule.h. */ int RM_GetApi(const char *funcname, void **targetPtrPtr) { + /* Lookup the requested module API and store the function pointer into the + * target pointer. The function returns REDISMODULE_ERR if there is no such + * named API, otherwise REDISMODULE_OK. + * + * This function is not meant to be used by modules developer, it is only + * used implicitly by including redismodule.h. */ dictEntry *he = dictFind(server.moduleapi, funcname); if (!he) return REDISMODULE_ERR; *targetPtrPtr = dictGetVal(he); @@ -599,17 +627,21 @@ void moduleHandlePropagationAfterCommandCallback(RedisModuleCtx *ctx) { /* We don't need to do anything here if the context was never used * in order to propagate commands. */ + if (!(ctx->flags & REDISMODULE_CTX_MULTI_EMITTED)) return; + + /* We don't need to do anything here if the server isn't inside + * a transaction. */ if (!server.propagate_in_transaction) return; - /* If this command is executed from with Lua or MULTI/EXEC we do noy + /* If this command is executed from with Lua or MULTI/EXEC we do not * need to propagate EXEC */ if (server.in_eval || server.in_exec) return; /* Handle the replication of the final EXEC, since whatever a command * emits is always wrapped around MULTI/EXEC. */ - beforePropagateMultiOrExec(0); alsoPropagate(server.execCommand,c->db->id,&shared.exec,1, PROPAGATE_AOF|PROPAGATE_REPL); + afterPropagateExec(); /* If this is not a module command context (but is instead a simple * callback context), we have to handle directly the "also propagate" @@ -708,6 +740,14 @@ int moduleGetCommandKeysViaAPI(struct redisCommand *cmd, robj **argv, int argc, return result->numkeys; } +/* -------------------------------------------------------------------------- + * ## Commands API + * + * These functions are used to implement custom Redis commands. + * + * For examples, see https://redis.io/topics/modules-intro. + * -------------------------------------------------------------------------- */ + /* Return non-zero if a module command, that was declared with the * flag "getkeys-api", is called in a special way to get the keys positions * and not to get executed. Otherwise zero is returned. */ @@ -882,11 +922,15 @@ int RM_CreateCommand(RedisModuleCtx *ctx, const char *name, RedisModuleCmdFunc c return REDISMODULE_OK; } -/* Called by RM_Init() to setup the `ctx->module` structure. - * - * This is an internal function, Redis modules developers don't need - * to use it. */ +/* -------------------------------------------------------------------------- + * ## Module information and time measurement + * -------------------------------------------------------------------------- */ + void RM_SetModuleAttribs(RedisModuleCtx *ctx, const char *name, int ver, int apiver) { + /* Called by RM_Init() to setup the `ctx->module` structure. + * + * This is an internal function, Redis modules developers don't need + * to use it. */ RedisModule *module; if (ctx->module != NULL) return; @@ -952,20 +996,29 @@ int RM_BlockedClientMeasureTimeEnd(RedisModuleBlockedClient *bc) { * repl-diskless-load to work if enabled. * The module should use RedisModule_IsIOError after reads, before using the * data that was read, and in case of error, propagate it upwards, and also be - * able to release the partially populated value and all it's allocations. */ + * able to release the partially populated value and all it's allocations. + * + * REDISMODULE_OPTION_NO_IMPLICIT_SIGNAL_MODIFIED: + * See RM_SignalModifiedKey(). + */ void RM_SetModuleOptions(RedisModuleCtx *ctx, int options) { ctx->module->options = options; } /* Signals that the key is modified from user's perspective (i.e. invalidate WATCH - * and client side caching). */ + * and client side caching). + * + * This is done automatically when a key opened for writing is closed, unless + * the option REDISMODULE_OPTION_NO_IMPLICIT_SIGNAL_MODIFIED has been set using + * RM_SetModuleOptions(). +*/ int RM_SignalModifiedKey(RedisModuleCtx *ctx, RedisModuleString *keyname) { signalModifiedKey(ctx->client,ctx->client->db,keyname); return REDISMODULE_OK; } /* -------------------------------------------------------------------------- - * Automatic memory management for modules + * ## Automatic memory management for modules * -------------------------------------------------------------------------- */ /* Enable automatic memory management. @@ -1061,7 +1114,7 @@ void autoMemoryCollect(RedisModuleCtx *ctx) { } /* -------------------------------------------------------------------------- - * String objects APIs + * ## String objects APIs * -------------------------------------------------------------------------- */ /* Create a new module string object. The returned string must be freed @@ -1330,14 +1383,6 @@ int RM_StringToLongDouble(const RedisModuleString *str, long double *ld) { * Returns REDISMODULE_OK on success and returns REDISMODULE_ERR if the string * is not a valid string representation of a stream ID. The special IDs "+" and * "-" are allowed. - * - * RedisModuleStreamID is a struct with two 64-bit fields, which is used in - * stream functions and defined as - * - * typedef struct RedisModuleStreamID { - * uint64_t ms; - * uint64_t seq; - * } RedisModuleStreamID; */ int RM_StringToStreamID(const RedisModuleString *str, RedisModuleStreamID *id) { streamID streamid; @@ -1392,13 +1437,15 @@ int RM_StringAppendBuffer(RedisModuleCtx *ctx, RedisModuleString *str, const cha } /* -------------------------------------------------------------------------- - * Reply APIs + * ## Reply APIs + * + * These functions are used for sending replies to the client. * * Most functions always return REDISMODULE_OK so you can use it with * 'return' in order to return from the command implementation with: * * if (... some condition ...) - * return RM_ReplyWithLongLong(ctx,mycount); + * return RedisModule_ReplyWithLongLong(ctx,mycount); * -------------------------------------------------------------------------- */ /* Send an error about the number of arguments given to the command, @@ -1687,7 +1734,7 @@ int RM_ReplyWithLongDouble(RedisModuleCtx *ctx, long double ld) { } /* -------------------------------------------------------------------------- - * Commands replication API + * ## Commands replication API * -------------------------------------------------------------------------- */ /* Helper function to replicate MULTI the first time we replicate something @@ -1696,7 +1743,7 @@ int RM_ReplyWithLongDouble(RedisModuleCtx *ctx, long double ld) { void moduleReplicateMultiIfNeeded(RedisModuleCtx *ctx) { /* Skip this if client explicitly wrap the command with MULTI, or if * the module command was called by a script. */ - if (server.lua_caller || server.in_exec) return; + if (server.in_eval || server.in_exec) return; /* If we already emitted MULTI return ASAP. */ if (server.propagate_in_transaction) return; /* If this is a thread safe context, we do not want to wrap commands @@ -1707,10 +1754,12 @@ void moduleReplicateMultiIfNeeded(RedisModuleCtx *ctx) { * context, we have to setup the op array for the "also propagate" API * so that RM_Replicate() will work. */ if (!(ctx->flags & REDISMODULE_CTX_MODULE_COMMAND_CALL)) { + serverAssert(ctx->saved_oparray.ops == NULL); ctx->saved_oparray = server.also_propagate; redisOpArrayInit(&server.also_propagate); } execCommandPropagateMulti(ctx->client->db->id); + ctx->flags |= REDISMODULE_CTX_MULTI_EMITTED; } /* Replicate the specified command and arguments to slaves and AOF, as effect @@ -1734,7 +1783,7 @@ void moduleReplicateMultiIfNeeded(RedisModuleCtx *ctx) { * the AOF or the replicas from the propagation of the specified command. * Otherwise, by default, the command will be propagated in both channels. * - * ## Note about calling this function from a thread safe context: + * #### Note about calling this function from a thread safe context: * * Normally when you call this function from the callback implementing a * module command, or any other callback provided by the Redis Module API, @@ -1746,7 +1795,7 @@ void moduleReplicateMultiIfNeeded(RedisModuleCtx *ctx) { * and the command specified is inserted in the AOF and replication stream * immediately. * - * ## Return value + * #### Return value * * The command returns REDISMODULE_ERR if the format specifiers are invalid * or the command name does not belong to a known command. */ @@ -1810,7 +1859,7 @@ int RM_ReplicateVerbatim(RedisModuleCtx *ctx) { } /* -------------------------------------------------------------------------- - * DB and Key APIs -- Generic API + * ## DB and Key APIs -- Generic API * -------------------------------------------------------------------------- */ /* Return the ID of the current client calling the currently active module @@ -2077,7 +2126,7 @@ int RM_GetContextFlags(RedisModuleCtx *ctx) { flags |= REDISMODULE_CTX_FLAGS_LOADING; /* Maxmemory and eviction policy */ - if (server.maxmemory > 0) { + if (server.maxmemory > 0 && (!server.masterhost || !server.repl_slave_ignore_maxmemory)) { flags |= REDISMODULE_CTX_FLAGS_MAXMEMORY; if (server.maxmemory_policy != MAXMEMORY_NO_EVICTION) @@ -2327,7 +2376,7 @@ mstime_t RM_GetExpire(RedisModuleKey *key) { * The function returns REDISMODULE_OK on success or REDISMODULE_ERR if * the key was not open for writing or is an empty key. */ int RM_SetExpire(RedisModuleKey *key, mstime_t expire) { - if (!(key->mode & REDISMODULE_WRITE) || key->value == NULL) + if (!(key->mode & REDISMODULE_WRITE) || key->value == NULL || (expire < 0 && expire != REDISMODULE_NO_EXPIRE)) return REDISMODULE_ERR; if (expire != REDISMODULE_NO_EXPIRE) { expire += mstime(); @@ -2338,6 +2387,36 @@ int RM_SetExpire(RedisModuleKey *key, mstime_t expire) { return REDISMODULE_OK; } +/* Return the key expire value, as absolute Unix timestamp. + * If no TTL is associated with the key or if the key is empty, + * REDISMODULE_NO_EXPIRE is returned. */ +mstime_t RM_GetAbsExpire(RedisModuleKey *key) { + mstime_t expire = getExpire(key->db,key->key); + if (expire == -1 || key->value == NULL) + return REDISMODULE_NO_EXPIRE; + return expire; +} + +/* Set a new expire for the key. If the special expire + * REDISMODULE_NO_EXPIRE is set, the expire is cancelled if there was + * one (the same as the PERSIST command). + * + * Note that the expire must be provided as a positive integer representing + * the absolute Unix timestamp the key should have. + * + * The function returns REDISMODULE_OK on success or REDISMODULE_ERR if + * the key was not open for writing or is an empty key. */ +int RM_SetAbsExpire(RedisModuleKey *key, mstime_t expire) { + if (!(key->mode & REDISMODULE_WRITE) || key->value == NULL || (expire < 0 && expire != REDISMODULE_NO_EXPIRE)) + return REDISMODULE_ERR; + if (expire != REDISMODULE_NO_EXPIRE) { + setExpire(key->ctx->client,key->db,key->key,expire); + } else { + removeExpire(key->db,key->key); + } + return REDISMODULE_OK; +} + /* Performs similar operation to FLUSHALL, and optionally start a new AOF file (if enabled) * If restart_aof is true, you must make sure the command that triggered this call is not * propagated to the AOF file. @@ -2361,7 +2440,9 @@ RedisModuleString *RM_RandomKey(RedisModuleCtx *ctx) { } /* -------------------------------------------------------------------------- - * Key API for String type + * ## Key API for String type + * + * See also RM_ValueLength(), which returns the length of a string. * -------------------------------------------------------------------------- */ /* If the key is open for writing, set the specified string 'str' as the @@ -2471,7 +2552,9 @@ int RM_StringTruncate(RedisModuleKey *key, size_t newlen) { } /* -------------------------------------------------------------------------- - * Key API for List type + * ## Key API for List type + * + * See also RM_ValueLength(), which returns the length of a list. * -------------------------------------------------------------------------- */ /* Push an element into a list, on head or tail depending on 'where' argument. @@ -2509,26 +2592,28 @@ RedisModuleString *RM_ListPop(RedisModuleKey *key, int where) { } /* -------------------------------------------------------------------------- - * Key API for Sorted Set type + * ## Key API for Sorted Set type + * + * See also RM_ValueLength(), which returns the length of a sorted set. * -------------------------------------------------------------------------- */ /* Conversion from/to public flags of the Modules API and our private flags, * so that we have everything decoupled. */ int moduleZsetAddFlagsToCoreFlags(int flags) { int retflags = 0; - if (flags & REDISMODULE_ZADD_XX) retflags |= ZADD_XX; - if (flags & REDISMODULE_ZADD_NX) retflags |= ZADD_NX; - if (flags & REDISMODULE_ZADD_GT) retflags |= ZADD_GT; - if (flags & REDISMODULE_ZADD_LT) retflags |= ZADD_LT; + if (flags & REDISMODULE_ZADD_XX) retflags |= ZADD_IN_XX; + if (flags & REDISMODULE_ZADD_NX) retflags |= ZADD_IN_NX; + if (flags & REDISMODULE_ZADD_GT) retflags |= ZADD_IN_GT; + if (flags & REDISMODULE_ZADD_LT) retflags |= ZADD_IN_LT; return retflags; } /* See previous function comment. */ int moduleZsetAddFlagsFromCoreFlags(int flags) { int retflags = 0; - if (flags & ZADD_ADDED) retflags |= REDISMODULE_ZADD_ADDED; - if (flags & ZADD_UPDATED) retflags |= REDISMODULE_ZADD_UPDATED; - if (flags & ZADD_NOP) retflags |= REDISMODULE_ZADD_NOP; + if (flags & ZADD_OUT_ADDED) retflags |= REDISMODULE_ZADD_ADDED; + if (flags & ZADD_OUT_UPDATED) retflags |= REDISMODULE_ZADD_UPDATED; + if (flags & ZADD_OUT_NOP) retflags |= REDISMODULE_ZADD_NOP; return retflags; } @@ -2565,16 +2650,16 @@ int moduleZsetAddFlagsFromCoreFlags(int flags) { * * 'score' double value is not a number (NaN). */ int RM_ZsetAdd(RedisModuleKey *key, double score, RedisModuleString *ele, int *flagsptr) { - int flags = 0; + int in_flags = 0, out_flags = 0; if (!(key->mode & REDISMODULE_WRITE)) return REDISMODULE_ERR; if (key->value && key->value->type != OBJ_ZSET) return REDISMODULE_ERR; if (key->value == NULL) moduleCreateEmptyKey(key,REDISMODULE_KEYTYPE_ZSET); - if (flagsptr) flags = moduleZsetAddFlagsToCoreFlags(*flagsptr); - if (zsetAdd(key->value,score,ele->ptr,&flags,NULL) == 0) { + if (flagsptr) in_flags = moduleZsetAddFlagsToCoreFlags(*flagsptr); + if (zsetAdd(key->value,score,ele->ptr,in_flags,&out_flags,NULL) == 0) { if (flagsptr) *flagsptr = 0; return REDISMODULE_ERR; } - if (flagsptr) *flagsptr = moduleZsetAddFlagsFromCoreFlags(flags); + if (flagsptr) *flagsptr = moduleZsetAddFlagsFromCoreFlags(out_flags); return REDISMODULE_OK; } @@ -2592,22 +2677,17 @@ int RM_ZsetAdd(RedisModuleKey *key, double score, RedisModuleString *ele, int *f * with the new score of the element after the increment, if no error * is returned. */ int RM_ZsetIncrby(RedisModuleKey *key, double score, RedisModuleString *ele, int *flagsptr, double *newscore) { - int flags = 0; + int in_flags = 0, out_flags = 0; if (!(key->mode & REDISMODULE_WRITE)) return REDISMODULE_ERR; if (key->value && key->value->type != OBJ_ZSET) return REDISMODULE_ERR; if (key->value == NULL) moduleCreateEmptyKey(key,REDISMODULE_KEYTYPE_ZSET); - if (flagsptr) flags = moduleZsetAddFlagsToCoreFlags(*flagsptr); - flags |= ZADD_INCR; - if (zsetAdd(key->value,score,ele->ptr,&flags,newscore) == 0) { + if (flagsptr) in_flags = moduleZsetAddFlagsToCoreFlags(*flagsptr); + in_flags |= ZADD_IN_INCR; + if (zsetAdd(key->value,score,ele->ptr,in_flags,&out_flags,newscore) == 0) { if (flagsptr) *flagsptr = 0; return REDISMODULE_ERR; } - /* zsetAdd() may signal back that the resulting score is not a number. */ - if (flagsptr && (*flagsptr & ZADD_NAN)) { - *flagsptr = 0; - return REDISMODULE_ERR; - } - if (flagsptr) *flagsptr = moduleZsetAddFlagsFromCoreFlags(flags); + if (flagsptr) *flagsptr = moduleZsetAddFlagsFromCoreFlags(out_flags); return REDISMODULE_OK; } @@ -2657,7 +2737,7 @@ int RM_ZsetScore(RedisModuleKey *key, RedisModuleString *ele, double *score) { } /* -------------------------------------------------------------------------- - * Key API for Sorted Set iterator + * ## Key API for Sorted Set iterator * -------------------------------------------------------------------------- */ void zsetKeyReset(RedisModuleKey *key) { @@ -2964,7 +3044,9 @@ int RM_ZsetRangePrev(RedisModuleKey *key) { } /* -------------------------------------------------------------------------- - * Key API for Hash type + * ## Key API for Hash type + * + * See also RM_ValueLength(), which returns the number of fields in a hash. * -------------------------------------------------------------------------- */ /* Set the field of the specified hash field to the specified value. @@ -3199,7 +3281,20 @@ int RM_HashGet(RedisModuleKey *key, int flags, ...) { } /* -------------------------------------------------------------------------- - * Key API for the stream type. + * ## Key API for Stream type + * + * For an introduction to streams, see https://redis.io/topics/streams-intro. + * + * The type RedisModuleStreamID, which is used in stream functions, is a struct + * with two 64-bit fields and is defined as + * + * typedef struct RedisModuleStreamID { + * uint64_t ms; + * uint64_t seq; + * } RedisModuleStreamID; + * + * See also RM_ValueLength(), which returns the length of a stream, and the + * conversion functions RM_StringToStreamID() and RM_CreateStringFromStreamID(). * -------------------------------------------------------------------------- */ /* Adds an entry to a stream. Like XADD without trimming. @@ -3366,8 +3461,8 @@ int RM_StreamDelete(RedisModuleKey *key, RedisModuleStreamID *id) { * // * // ... Do stuff ... * // - * RedisModule_Free(field); - * RedisModule_Free(value); + * RedisModule_FreeString(ctx, field); + * RedisModule_FreeString(ctx, value); * } * } * RedisModule_StreamIteratorStop(key); @@ -3648,7 +3743,9 @@ long long RM_StreamTrimByID(RedisModuleKey *key, int flags, RedisModuleStreamID } /* -------------------------------------------------------------------------- - * Redis <-> Modules generic Call() API + * ## Calling Redis commands from modules + * + * RM_Call() sends a command to Redis. The remaining functions handle the reply. * -------------------------------------------------------------------------- */ /* Create a new RedisModuleCallReply object. The processing of the reply @@ -4067,20 +4164,30 @@ RedisModuleCallReply *RM_Call(RedisModuleCtx *ctx, const char *cmdname, const ch } } - /* If we are using single commands replication, we need to wrap what - * we propagate into a MULTI/EXEC block, so that it will be atomic like - * a Lua script in the context of AOF and slaves. */ - if (replicate) moduleReplicateMultiIfNeeded(ctx); + /* We need to use a global replication_allowed flag in order to prevent + * replication of nested RM_Calls. Example: + * 1. module1.foo does RM_Call of module2.bar without replication (i.e. no '!') + * 2. module2.bar internally calls RM_Call of INCR with '!' + * 3. at the end of module1.foo we call RM_ReplicateVerbatim + * We want the replica/AOF to see only module1.foo and not the INCR from module2.bar */ + int prev_replication_allowed = server.replication_allowed; + server.replication_allowed = replicate && server.replication_allowed; /* Run the command */ int call_flags = CMD_CALL_SLOWLOG | CMD_CALL_STATS | CMD_CALL_NOWRAP; if (replicate) { + /* If we are using single commands replication, we need to wrap what + * we propagate into a MULTI/EXEC block, so that it will be atomic like + * a Lua script in the context of AOF and slaves. */ + moduleReplicateMultiIfNeeded(ctx); + if (!(flags & REDISMODULE_ARGV_NO_AOF)) call_flags |= CMD_CALL_PROPAGATE_AOF; if (!(flags & REDISMODULE_ARGV_NO_REPLICAS)) call_flags |= CMD_CALL_PROPAGATE_REPL; } call(c,call_flags); + server.replication_allowed = prev_replication_allowed; serverAssert((c->flags & CLIENT_BLOCKED) == 0); @@ -4121,7 +4228,7 @@ const char *RM_CallReplyProto(RedisModuleCallReply *reply, size_t *len) { } /* -------------------------------------------------------------------------- - * Modules data types + * ## Modules data types * * When String DMA or using existing data structures is not enough, it is * possible to create new data types from scratch and export them to @@ -4264,6 +4371,12 @@ void moduleTypeNameByID(char *name, uint64_t moduleid) { } } +/* Return the name of the module that owns the specified moduleType. */ +const char *moduleTypeModuleName(moduleType *mt) { + if (!mt || !mt->module) return NULL; + return mt->module->name; +} + /* Create a copy of a module type value using the copy callback. If failed * or not supported, produce an error reply and return NULL. */ @@ -4479,7 +4592,7 @@ void *RM_ModuleTypeGetValue(RedisModuleKey *key) { } /* -------------------------------------------------------------------------- - * RDB loading and saving functions + * ## RDB loading and saving functions * -------------------------------------------------------------------------- */ /* Called when there is a load error in the context of a module. On some @@ -4791,7 +4904,7 @@ ssize_t rdbSaveModulesAux(rio *rdb, int when) { } /* -------------------------------------------------------------------------- - * Key digest API (DEBUG DIGEST interface for modules types) + * ## Key digest API (DEBUG DIGEST interface for modules types) * -------------------------------------------------------------------------- */ /* Add a new element to the digest. This function can be called multiple times @@ -4912,7 +5025,7 @@ RedisModuleString *RM_SaveDataTypeToString(RedisModuleCtx *ctx, void *data, cons } /* -------------------------------------------------------------------------- - * AOF API for modules data types + * ## AOF API for modules data types * -------------------------------------------------------------------------- */ /* Emits a command into the AOF during the AOF rewriting process. This function @@ -4967,7 +5080,7 @@ void RM_EmitAOF(RedisModuleIO *io, const char *cmdname, const char *fmt, ...) { } /* -------------------------------------------------------------------------- - * IO context handling + * ## IO context handling * -------------------------------------------------------------------------- */ RedisModuleCtx *RM_GetContextFromIO(RedisModuleIO *io) { @@ -4994,7 +5107,7 @@ const RedisModuleString *RM_GetKeyNameFromModuleKey(RedisModuleKey *key) { } /* -------------------------------------------------------------------------- - * Logging + * ## Logging * -------------------------------------------------------------------------- */ /* This is the low level function implementing both: @@ -5025,10 +5138,10 @@ void moduleLogRaw(RedisModule *module, const char *levelstr, const char *fmt, va * printf-alike specifiers, while level is a string describing the log * level to use when emitting the log, and must be one of the following: * - * * "debug" - * * "verbose" - * * "notice" - * * "warning" + * * "debug" (`REDISMODULE_LOGLEVEL_DEBUG`) + * * "verbose" (`REDISMODULE_LOGLEVEL_VERBOSE`) + * * "notice" (`REDISMODULE_LOGLEVEL_NOTICE`) + * * "warning" (`REDISMODULE_LOGLEVEL_WARNING`) * * If the specified log level is invalid, verbose is used by default. * There is a fixed limit to the length of the log line this function is able @@ -5079,7 +5192,10 @@ void RM_LatencyAddSample(const char *event, mstime_t latency) { } /* -------------------------------------------------------------------------- - * Blocking clients from modules + * ## Blocking clients from modules + * + * For a guide about blocking commands in modules, see + * https://redis.io/topics/modules-blocking-ops. * -------------------------------------------------------------------------- */ /* Readable handler for the awake pipe. We do nothing here, the awake bytes @@ -5140,11 +5256,6 @@ void unblockClientFromModule(client *c) { moduleUnblockClient(c); bc->client = NULL; - /* Reset the client for a new query since, for blocking commands implemented - * into modules, we do not it immediately after the command returns (and - * the client blocks) in order to be still able to access the argument - * vector from callbacks. */ - resetClient(c); } /* Block a client in the context of a module: this function implements both @@ -5544,6 +5655,12 @@ void moduleHandleBlockedClients(void) { * API to unblock the client and the memory will be released. */ void moduleBlockedClientTimedOut(client *c) { RedisModuleBlockedClient *bc = c->bpop.module_blocked_handle; + + /* Protect against re-processing: don't serve clients that are already + * in the unblocking list for any reason (including RM_UnblockClient() + * explicit call). See #6798. */ + if (bc->unblocked) return; + RedisModuleCtx ctx = REDISMODULE_CTX_INIT; ctx.flags |= REDISMODULE_CTX_BLOCKED_TIMEOUT; ctx.module = bc->module; @@ -5600,7 +5717,7 @@ int RM_BlockedClientDisconnected(RedisModuleCtx *ctx) { } /* -------------------------------------------------------------------------- - * Thread Safe Contexts + * ## Thread Safe Contexts * -------------------------------------------------------------------------- */ /* Return a context which can be used inside threads to make Redis context @@ -5710,7 +5827,7 @@ void moduleReleaseGIL(void) { /* -------------------------------------------------------------------------- - * Module Keyspace Notifications API + * ## Module Keyspace Notifications API * -------------------------------------------------------------------------- */ /* Subscribe to keyspace notifications. This is a low-level version of the @@ -5734,6 +5851,7 @@ void moduleReleaseGIL(void) { * - REDISMODULE_NOTIFY_EXPIRED: Expiration events * - REDISMODULE_NOTIFY_EVICTED: Eviction events * - REDISMODULE_NOTIFY_STREAM: Stream events + * - REDISMODULE_NOTIFY_MODULE: Module types events * - REDISMODULE_NOTIFY_KEYMISS: Key-miss events * - REDISMODULE_NOTIFY_ALL: All events (Excluding REDISMODULE_NOTIFY_KEYMISS) * - REDISMODULE_NOTIFY_LOADED: A special notification available only for modules, @@ -5843,7 +5961,7 @@ void moduleUnsubscribeNotifications(RedisModule *module) { } /* -------------------------------------------------------------------------- - * Modules Cluster API + * ## Modules Cluster API * -------------------------------------------------------------------------- */ /* The Cluster message callback function pointer type. */ @@ -6098,7 +6216,7 @@ void RM_SetClusterFlags(RedisModuleCtx *ctx, uint64_t flags) { } /* -------------------------------------------------------------------------- - * Modules Timers API + * ## Modules Timers API * * Module timers are an high precision "green timers" abstraction where * every module can register even millions of timers without problems, even if @@ -6272,7 +6390,7 @@ int RM_GetTimerInfo(RedisModuleCtx *ctx, RedisModuleTimerID id, uint64_t *remain } /* -------------------------------------------------------------------------- - * Modules ACL API + * ## Modules ACL API * * Implements a hook into the authentication and authorization within Redis. * --------------------------------------------------------------------------*/ @@ -6502,7 +6620,7 @@ RedisModuleString *RM_GetClientCertificate(RedisModuleCtx *ctx, uint64_t client_ } /* -------------------------------------------------------------------------- - * Modules Dictionary API + * ## Modules Dictionary API * * Implements a sorted dictionary (actually backed by a radix tree) with * the usual get / set / del / num-items API, together with an iterator @@ -6756,7 +6874,7 @@ int RM_DictCompare(RedisModuleDictIter *di, const char *op, RedisModuleString *k /* -------------------------------------------------------------------------- - * Modules Info fields + * ## Modules Info fields * -------------------------------------------------------------------------- */ int RM_InfoEndDictField(RedisModuleInfoCtx *ctx); @@ -7070,7 +7188,7 @@ double RM_ServerInfoGetFieldDouble(RedisModuleServerInfoData *data, const char* } /* -------------------------------------------------------------------------- - * Modules utility APIs + * ## Modules utility APIs * -------------------------------------------------------------------------- */ /* Return random bytes using SHA1 in counter mode with a /dev/urandom @@ -7089,7 +7207,7 @@ void RM_GetRandomHexChars(char *dst, size_t len) { } /* -------------------------------------------------------------------------- - * Modules API exporting / importing + * ## Modules API exporting / importing * -------------------------------------------------------------------------- */ /* This function is called by a module in order to export some API with a @@ -7226,7 +7344,7 @@ int moduleUnregisterFilters(RedisModule *module) { } /* -------------------------------------------------------------------------- - * Module Command Filter API + * ## Module Command Filter API * -------------------------------------------------------------------------- */ /* Register a new command filter function. @@ -7436,7 +7554,7 @@ float RM_GetUsedMemoryRatio(){ } /* -------------------------------------------------------------------------- - * Scanning keyspace and hashes + * ## Scanning keyspace and hashes * -------------------------------------------------------------------------- */ typedef void (*RedisModuleScanCB)(RedisModuleCtx *ctx, RedisModuleString *keyname, RedisModuleKey *key, void *privdata); @@ -7707,7 +7825,7 @@ int RM_ScanKey(RedisModuleKey *key, RedisModuleScanCursor *cursor, RedisModuleSc /* -------------------------------------------------------------------------- - * Module fork API + * ## Module fork API * -------------------------------------------------------------------------- */ /* Create a background child process with the current frozen snaphost of the @@ -7767,7 +7885,7 @@ int TerminateModuleForkChild(int child_pid, int wait) { serverLog(LL_VERBOSE,"Killing running module fork child: %ld", (long) server.child_pid); if (kill(server.child_pid,SIGUSR1) != -1 && wait) { - while(wait4(server.child_pid,&statloc,0,NULL) != + while(waitpid(server.child_pid, &statloc, 0) != server.child_pid); } /* Reset the buffer accumulating changes while the child saves. */ @@ -7801,7 +7919,7 @@ void ModuleForkDoneHandler(int exitcode, int bysignal) { } /* -------------------------------------------------------------------------- - * Server hooks implementation + * ## Server hooks implementation * -------------------------------------------------------------------------- */ /* Register to be notified, via a callback, when the specified server event @@ -8682,6 +8800,10 @@ size_t moduleCount(void) { return dictSize(modules); } +/* -------------------------------------------------------------------------- + * ## Key eviction API + * -------------------------------------------------------------------------- */ + /* Set the key last access time for LRU based eviction. not relevant if the * servers's maxmemory policy is LFU based. Value is idle time in milliseconds. * returns REDISMODULE_OK if the LRU was updated, REDISMODULE_ERR otherwise. */ @@ -8732,6 +8854,10 @@ int RM_GetLFU(RedisModuleKey *key, long long *lfu_freq) { return REDISMODULE_OK; } +/* -------------------------------------------------------------------------- + * ## Miscellaneous APIs + * -------------------------------------------------------------------------- */ + /** * Returns the full ContextFlags mask, using the return value * the module can check if a certain set of flags are supported @@ -8880,6 +9006,10 @@ int *RM_GetCommandKeys(RedisModuleCtx *ctx, RedisModuleString **argv, int argc, return res; } +/* -------------------------------------------------------------------------- + * ## Defrag API + * -------------------------------------------------------------------------- */ + /* The defrag context, used to manage state during calls to the data type * defrag callback. */ diff --git a/src/modules/gendoc.rb b/src/modules/gendoc.rb index 2fd2ec5d7..f83b1ad9d 100644 --- a/src/modules/gendoc.rb +++ b/src/modules/gendoc.rb @@ -1,3 +1,4 @@ +# coding: utf-8 # gendoc.rb -- Converts the top-comments inside module.c to modules API # reference documentation in markdown format. @@ -21,15 +22,48 @@ def markdown(s) l = l.gsub(/(?\n\n" + puts "### `#{name}`\n\n" puts " #{proto}\n" comment = "" while true @@ -50,13 +88,87 @@ def docufy(src,i) puts comment+"\n\n" end +# Print a comment from line until */ is found, as markdown. +def section_doc(src, i) + name = get_section_heading(src, i) + comment = "\n\n" + while true + # append line, except if it's a horizontal divider + comment = comment + src[i] if src[i] !~ /^[\/ ]?\*{1,2} ?-{50,}/ + break if src[i] =~ /\*\// + i = i+1 + end + comment = markdown(comment) + puts comment+"\n\n" +end + +# generates an id suitable for links within the page +def section_name_to_id(name) + return "section-" + + name.strip.downcase.gsub(/[^a-z0-9]+/, '-').gsub(/^-+|-+$/, '') +end + +# Returns the name of the first section heading in the comment block for which +# is_section_doc(src, i) is true +def get_section_heading(src, i) + if src[i] =~ /^\/\*\*? \#+ *(.*)/ + heading = $1 + elsif src[i+1] =~ /^ ?\* \#+ *(.*)/ + heading = $1 + end + return heading.gsub(' -- ', ' – ') +end + +# Returns true if the line is the start of a generic documentation section. Such +# section must start with the # symbol, i.e. a markdown heading, on the first or +# the second line. +def is_section_doc(src, i) + return src[i] =~ /^\/\*\*? \#/ || + (src[i] =~ /^\/\*/ && src[i+1] =~ /^ ?\* \#/) +end + +def is_func_line(src, i) + line = src[i] + return line =~ /RM_/ && + line[0] != ' ' && line[0] != '#' && line[0] != '/' && + src[i-1] =~ /\*\// +end + puts "# Modules API reference\n\n" puts "\n\n" -src = File.open("../module.c").to_a -src.each_with_index{|line,i| - if line =~ /RM_/ && line[0] != ' ' && line[0] != '#' && line[0] != '/' - if src[i-1] =~ /\*\// - docufy(src,i) - end +src = File.open(File.dirname(__FILE__) ++ "/../module.c").to_a + +# Build function index +$index = {} +src.each_with_index do |line,i| + if is_func_line(src, i) + line =~ /RM_([A-z0-9]+)/ + name = "RedisModule_#{$1}" + $index[name] = true end -} +end + +# Print TOC +puts "## Sections\n\n" +src.each_with_index do |_line,i| + if is_section_doc(src, i) + name = get_section_heading(src, i) + puts "* [#{name}](\##{section_name_to_id(name)})\n" + end +end +puts "* [Function index](#section-function-index)\n\n" + +# Docufy: Print function prototype and markdown docs +src.each_with_index do |_line,i| + if is_func_line(src, i) + docufy(src, i) + elsif is_section_doc(src, i) + section_doc(src, i) + end +end + +# Print function index +puts "\n\n" +puts "## Function index\n\n" +$index.keys.sort.each{|x| puts "* [`#{x}`](\##{x})\n"} +puts "\n" diff --git a/src/multi.c b/src/multi.c index d88c5f1b8..902c919c7 100644 --- a/src/multi.c +++ b/src/multi.c @@ -113,34 +113,34 @@ void discardCommand(client *c) { addReply(c,shared.ok); } -void beforePropagateMultiOrExec(int multi) { - if (multi) { - /* Propagating MULTI */ - serverAssert(!server.propagate_in_transaction); - server.propagate_in_transaction = 1; - } else { - /* Propagating EXEC */ - serverAssert(server.propagate_in_transaction == 1); - server.propagate_in_transaction = 0; - } +void beforePropagateMulti() { + /* Propagating MULTI */ + serverAssert(!server.propagate_in_transaction); + server.propagate_in_transaction = 1; +} + +void afterPropagateExec() { + /* Propagating EXEC */ + serverAssert(server.propagate_in_transaction == 1); + server.propagate_in_transaction = 0; } /* Send a MULTI command to all the slaves and AOF file. Check the execCommand * implementation for more information. */ void execCommandPropagateMulti(int dbid) { - beforePropagateMultiOrExec(1); + beforePropagateMulti(); propagate(server.multiCommand,dbid,&shared.multi,1, PROPAGATE_AOF|PROPAGATE_REPL); } void execCommandPropagateExec(int dbid) { - beforePropagateMultiOrExec(0); propagate(server.execCommand,dbid,&shared.exec,1, PROPAGATE_AOF|PROPAGATE_REPL); + afterPropagateExec(); } /* Aborts a transaction, with a specific error message. - * The transaction is always aboarted with -EXECABORT so that the client knows + * The transaction is always aborted with -EXECABORT so that the client knows * the server exited the multi state, but the actual reason for the abort is * included too. * Note: 'error' may or may not end with \r\n. see addReplyErrorFormat. */ @@ -202,11 +202,9 @@ void execCommand(client *c) { c->cmd = c->mstate.commands[j].cmd; /* ACL permissions are also checked at the time of execution in case - * they were changed after the commands were ququed. */ + * they were changed after the commands were queued. */ int acl_errpos; - int acl_retval = ACLCheckCommandPerm(c,&acl_errpos); - if (acl_retval == ACL_OK && c->cmd->proc == publishCommand) - acl_retval = ACLCheckPubsubPerm(c,1,1,0,&acl_errpos); + int acl_retval = ACLCheckAllPerm(c,&acl_errpos); if (acl_retval != ACL_OK) { char *reason; switch (acl_retval) { @@ -217,7 +215,8 @@ void execCommand(client *c) { reason = "no permission to touch the specified keys"; break; case ACL_DENIED_CHANNEL: - reason = "no permission to publish to the specified channel"; + reason = "no permission to access one of the channels used " + "as arguments"; break; default: reason = "no permission"; @@ -254,7 +253,6 @@ void execCommand(client *c) { if (server.propagate_in_transaction) { int is_master = server.masterhost == NULL; server.dirty++; - beforePropagateMultiOrExec(0); /* If inside the MULTI/EXEC block this instance was suddenly * switched from master to slave (using the SLAVEOF command), the * initial MULTI was propagated into the replication backlog, but the @@ -264,6 +262,7 @@ void execCommand(client *c) { char *execcmd = "*1\r\n$4\r\nEXEC\r\n"; feedReplicationBacklog(execcmd,strlen(execcmd)); } + afterPropagateExec(); } server.in_exec = 0; diff --git a/src/networking.c b/src/networking.c index 50e4b71bc..2355a376b 100644 --- a/src/networking.c +++ b/src/networking.c @@ -154,6 +154,7 @@ client *createClient(connection *conn) { c->read_reploff = 0; c->repl_ack_off = 0; c->repl_ack_time = 0; + c->repl_last_partial_write = 0; c->slave_listening_port = 0; c->slave_addr = NULL; c->slave_capa = SLAVE_CAPA_NONE; @@ -331,8 +332,9 @@ void _addReplyProtoToList(client *c, const char *s, size_t len) { memcpy(tail->buf, s, len); listAddNodeTail(c->reply, tail); c->reply_bytes += tail->size; + + asyncCloseClientOnOutputBufferLimitReached(c); } - asyncCloseClientOnOutputBufferLimitReached(c); } /* ----------------------------------------------------------------------------- @@ -562,7 +564,7 @@ void *addReplyDeferredLen(client *c) { void setDeferredReply(client *c, void *node, const char *s, size_t length) { listNode *ln = (listNode*)node; - clientReplyBlock *next; + clientReplyBlock *next, *prev; /* Abort when *node is NULL: when the client should not accept writes * we return NULL in addReplyDeferredLen() */ @@ -571,14 +573,31 @@ void setDeferredReply(client *c, void *node, const char *s, size_t length) { /* Normally we fill this dummy NULL node, added by addReplyDeferredLen(), * with a new buffer structure containing the protocol needed to specify - * the length of the array following. However sometimes when there is - * little memory to move, we may instead remove this NULL node, and prefix - * our protocol in the node immediately after to it, in order to save a - * write(2) syscall later. Conditions needed to do it: + * the length of the array following. However sometimes there might be room + * in the previous/next node so we can instead remove this NULL node, and + * suffix/prefix our data in the node immediately before/after it, in order + * to save a write(2) syscall later. Conditions needed to do it: * + * - The prev node is non-NULL and has space in it or * - The next node is non-NULL, * - It has enough room already allocated * - And not too large (avoid large memmove) */ + if (ln->prev != NULL && (prev = listNodeValue(ln->prev)) && + prev->size - prev->used > 0) + { + size_t len_to_copy = prev->size - prev->used; + if (len_to_copy > length) + len_to_copy = length; + memcpy(prev->buf + prev->used, s, len_to_copy); + prev->used += len_to_copy; + length -= len_to_copy; + if (length == 0) { + listDelNode(c->reply, ln); + return; + } + s += len_to_copy; + } + if (ln->next != NULL && (next = listNodeValue(ln->next)) && next->size - next->used >= length && next->used < PROTO_REPLY_CHUNK_BYTES * 4) @@ -596,8 +615,9 @@ void setDeferredReply(client *c, void *node, const char *s, size_t length) { memcpy(buf->buf, s, length); listNodeValue(ln) = buf; c->reply_bytes += buf->size; + + asyncCloseClientOnOutputBufferLimitReached(c); } - asyncCloseClientOnOutputBufferLimitReached(c); } /* Populate the length object and try gluing it to the next chunk. */ @@ -1531,9 +1551,7 @@ int writeToClient(client *c, int handler_installed) { } atomicIncr(server.stat_net_output_bytes, totwritten); if (nwritten == -1) { - if (connGetState(c->conn) == CONN_STATE_CONNECTED) { - nwritten = 0; - } else { + if (connGetState(c->conn) != CONN_STATE_CONNECTED) { serverLog(LL_VERBOSE, "Error writing to client: %s", connGetLastError(c->conn)); freeClientAsync(c); @@ -1645,6 +1663,9 @@ void resetClient(client *c) { c->flags |= CLIENT_REPLY_SKIP; c->flags &= ~CLIENT_REPLY_SKIP_NEXT; } + + /* Always clear the prevent logging field. */ + c->flags &= ~CLIENT_PREVENT_LOGGING; } /* This function is used when we want to re-enter the event loop but there @@ -1954,13 +1975,10 @@ void commandProcessed(client *c) { c->reploff = c->read_reploff - sdslen(c->querybuf) + c->qb_pos; } - /* Don't reset the client structure for clients blocked in a - * module blocking command, so that the reply callback will - * still be able to access the client argv and argc field. - * The client will be reset in unblockClientFromModule(). */ - if (!(c->flags & CLIENT_BLOCKED) || - (c->btype != BLOCKED_MODULE && c->btype != BLOCKED_PAUSE)) - { + /* Don't reset the client structure for blocked clients, so that the reply + * callback will still be able to access the client argv and argc fields. + * The client will be reset in unblockClient(). */ + if (!(c->flags & CLIENT_BLOCKED)) { resetClient(c); } @@ -1990,12 +2008,20 @@ void commandProcessed(client *c) { * of processing the command, otherwise C_OK is returned. */ int processCommandAndResetClient(client *c) { int deadclient = 0; + client *old_client = server.current_client; server.current_client = c; if (processCommand(c) == C_OK) { commandProcessed(c); } if (server.current_client == NULL) deadclient = 1; - server.current_client = NULL; + /* + * Restore the old client, this is needed because when a script + * times out, we will get into this code from processEventsWhileBlocked. + * Which will cause to set the server.current_client. If not restored + * we will return 1 to our caller which will falsely indicate the client + * is dead and will stop reading from its buffer. + */ + server.current_client = old_client; /* performEvictions may flush slave output buffers. This may * result in a slave, that may be the active client, to be * freed. */ @@ -2941,6 +2967,7 @@ void helloCommand(client *c) { int moreargs = (c->argc-1) - j; const char *opt = c->argv[j]->ptr; if (!strcasecmp(opt,"AUTH") && moreargs >= 2) { + preventCommandLogging(c); if (ACLAuthenticateUser(c, c->argv[j+1], c->argv[j+2]) == C_ERR) { addReplyError(c,"-WRONGPASS invalid username-password pair or user is disabled."); return; @@ -3007,7 +3034,7 @@ void securityWarningCommand(client *c) { static time_t logged_time; time_t now = time(NULL); - if (labs(now-logged_time) > 60) { + if (llabs(now-logged_time) > 60) { serverLog(LL_WARNING,"Possible SECURITY ATTACK detected. It looks like somebody is sending POST or Host: commands to Redis. This is likely due to an attacker attempting to use Cross Protocol Scripting to compromise your Redis instance. Connection aborted."); logged_time = now; } @@ -3304,6 +3331,8 @@ int areClientsPaused(void) { * if it has. Also returns true if clients are now paused and false * otherwise. */ int checkClientPauseTimeoutAndReturnIfPaused(void) { + if (!areClientsPaused()) + return 0; if (server.client_pause_end_time < server.mstime) { unpauseClients(); } @@ -3332,7 +3361,7 @@ void processEventsWhileBlocked(void) { /* Note: when we are processing events while blocked (for instance during * busy Lua scripts), we set a global flag. When such flag is set, we * avoid handling the read part of clients using threaded I/O. - * See https://github.com/antirez/redis/issues/6988 for more info. */ + * See https://github.com/redis/redis/issues/6988 for more info. */ ProcessingEventsWhileBlocked = 1; while (iterations--) { long long startval = server.events_processed_while_blocked; diff --git a/src/notify.c b/src/notify.c index 5c7634bce..afaddbfca 100644 --- a/src/notify.c +++ b/src/notify.c @@ -56,6 +56,7 @@ int keyspaceEventsStringToFlags(char *classes) { case 'E': flags |= NOTIFY_KEYEVENT; break; case 't': flags |= NOTIFY_STREAM; break; case 'm': flags |= NOTIFY_KEY_MISS; break; + case 'd': flags |= NOTIFY_MODULE; break; default: return -1; } } @@ -82,6 +83,7 @@ sds keyspaceEventsFlagsToString(int flags) { if (flags & NOTIFY_EXPIRED) res = sdscatlen(res,"x",1); if (flags & NOTIFY_EVICTED) res = sdscatlen(res,"e",1); if (flags & NOTIFY_STREAM) res = sdscatlen(res,"t",1); + if (flags & NOTIFY_MODULE) res = sdscatlen(res,"d",1); } if (flags & NOTIFY_KEYSPACE) res = sdscatlen(res,"K",1); if (flags & NOTIFY_KEYEVENT) res = sdscatlen(res,"E",1); diff --git a/src/object.c b/src/object.c index ce2f3dc14..b75e547b9 100644 --- a/src/object.c +++ b/src/object.c @@ -727,7 +727,11 @@ int getRangeLongFromObjectOrReply(client *c, robj *o, long min, long max, long * } int getPositiveLongFromObjectOrReply(client *c, robj *o, long *target, const char *msg) { - return getRangeLongFromObjectOrReply(c, o, 0, LONG_MAX, target, msg); + if (msg) { + return getRangeLongFromObjectOrReply(c, o, 0, LONG_MAX, target, msg); + } else { + return getRangeLongFromObjectOrReply(c, o, 0, LONG_MAX, target, "value is out of range, must be positive"); + } } int getIntFromObjectOrReply(client *c, robj *o, int *target, const char *msg) { diff --git a/src/pubsub.c b/src/pubsub.c index 5f7335bbe..3409deac2 100644 --- a/src/pubsub.c +++ b/src/pubsub.c @@ -331,21 +331,6 @@ int pubsubPublishMessage(robj *channel, robj *message) { return receivers; } -/* This wraps handling ACL channel permissions for the given client. */ -int pubsubCheckACLPermissionsOrReply(client *c, int idx, int count, int literal) { - /* Check if the user can run the command according to the current - * ACLs. */ - int acl_chanpos; - int acl_retval = ACLCheckPubsubPerm(c,idx,count,literal,&acl_chanpos); - if (acl_retval == ACL_DENIED_CHANNEL) { - addACLLogEntry(c,acl_retval,acl_chanpos,NULL); - addReplyError(c, - "-NOPERM this user has no permissions to access " - "one of the channels used as arguments"); - } - return acl_retval; -} - /*----------------------------------------------------------------------------- * Pubsub commands implementation *----------------------------------------------------------------------------*/ @@ -353,7 +338,6 @@ int pubsubCheckACLPermissionsOrReply(client *c, int idx, int count, int literal) /* SUBSCRIBE channel [channel ...] */ void subscribeCommand(client *c) { int j; - if (pubsubCheckACLPermissionsOrReply(c,1,c->argc-1,0) != ACL_OK) return; if ((c->flags & CLIENT_DENY_BLOCKING) && !(c->flags & CLIENT_MULTI)) { /** * A client that has CLIENT_DENY_BLOCKING flag on @@ -387,7 +371,6 @@ void unsubscribeCommand(client *c) { /* PSUBSCRIBE pattern [pattern ...] */ void psubscribeCommand(client *c) { int j; - if (pubsubCheckACLPermissionsOrReply(c,1,c->argc-1,1) != ACL_OK) return; if ((c->flags & CLIENT_DENY_BLOCKING) && !(c->flags & CLIENT_MULTI)) { /** * A client that has CLIENT_DENY_BLOCKING flag on @@ -420,7 +403,6 @@ void punsubscribeCommand(client *c) { /* PUBLISH */ void publishCommand(client *c) { - if (pubsubCheckACLPermissionsOrReply(c,1,1,0) != ACL_OK) return; int receivers = pubsubPublishMessage(c->argv[1],c->argv[2]); if (server.cluster_enabled) clusterPropagatePublish(c->argv[1],c->argv[2]); diff --git a/src/quicklist.c b/src/quicklist.c index 7b7aa7839..5a1e41dcc 100644 --- a/src/quicklist.c +++ b/src/quicklist.c @@ -315,7 +315,9 @@ REDIS_STATIC void __quicklistCompress(const quicklist *quicklist, if (forward == node || reverse == node) in_depth = 1; - if (forward == reverse) + /* We passed into compress depth of opposite side of the quicklist + * so there's no need to compress anything and we can exit. */ + if (forward == reverse || forward->next == reverse) return; forward = forward->next; @@ -325,11 +327,9 @@ REDIS_STATIC void __quicklistCompress(const quicklist *quicklist, if (!in_depth) quicklistCompressNode(node); - if (depth > 2) { - /* At this point, forward and reverse are one node beyond depth */ - quicklistCompressNode(forward); - quicklistCompressNode(reverse); - } + /* At this point, forward and reverse are one node beyond depth */ + quicklistCompressNode(forward); + quicklistCompressNode(reverse); } #define quicklistCompress(_ql, _node) \ @@ -380,10 +380,11 @@ REDIS_STATIC void __quicklistInsertNode(quicklist *quicklist, quicklist->head = quicklist->tail = new_node; } + /* Update len first, so in __quicklistCompress we know exactly len */ + quicklist->len++; + if (old_node) quicklistCompress(quicklist, old_node); - - quicklist->len++; } /* Wrappers for node inserting around existing node. */ @@ -602,15 +603,16 @@ REDIS_STATIC void __quicklistDelNode(quicklist *quicklist, quicklist->head = node->next; } + /* Update len first, so in __quicklistCompress we know exactly len */ + quicklist->len--; + quicklist->count -= node->count; + /* If we deleted a node within our compress depth, we * now have compressed nodes needing to be decompressed. */ __quicklistCompress(quicklist, NULL); - quicklist->count -= node->count; - zfree(node->zl); zfree(node); - quicklist->len--; } /* Delete one entry from list given the node for the entry and a pointer @@ -1296,17 +1298,24 @@ void quicklistRotate(quicklist *quicklist) { /* First, get the tail entry */ unsigned char *p = ziplistIndex(quicklist->tail->zl, -1); - unsigned char *value; + unsigned char *value, *tmp; long long longval; unsigned int sz; char longstr[32] = {0}; - ziplistGet(p, &value, &sz, &longval); + ziplistGet(p, &tmp, &sz, &longval); /* If value found is NULL, then ziplistGet populated longval instead */ - if (!value) { + if (!tmp) { /* Write the longval as a string so we can re-add it */ sz = ll2string(longstr, sizeof(longstr), longval); value = (unsigned char *)longstr; + } else if (quicklist->len == 1) { + /* Copy buffer since there could be a memory overlap when move + * entity from tail to head in the same ziplist. */ + value = zmalloc(sz); + memcpy(value, tmp, sz); + } else { + value = tmp; } /* Add tail entry to head (must happen before tail is deleted). */ @@ -1321,6 +1330,8 @@ void quicklistRotate(quicklist *quicklist) { /* Remove tail entry. */ quicklistDelIndex(quicklist, quicklist->tail, &p); + if (value != (unsigned char*)longstr && value != tmp) + zfree(value); } /* pop from quicklist and return result in 'data' ptr. Value of 'data' @@ -1509,8 +1520,6 @@ void quicklistBookmarksClear(quicklist *ql) { #define yell(str, ...) printf("ERROR! " str "\n\n", __VA_ARGS__) -#define OK printf("\tOK\n") - #define ERROR \ do { \ printf("\tERROR!\n"); \ @@ -1630,7 +1639,6 @@ static int _ql_verify(quicklist *ql, uint32_t len, uint32_t count, } if (ql->len == 0 && !errors) { - OK; return errors; } @@ -1679,8 +1687,6 @@ static int _ql_verify(quicklist *ql, uint32_t len, uint32_t count, } } - if (!errors) - OK; return errors; } @@ -1692,9 +1698,10 @@ static char *genstr(char *prefix, int i) { } /* main test, but callable from other files */ -int quicklistTest(int argc, char *argv[]) { +int quicklistTest(int argc, char *argv[], int accurate) { UNUSED(argc); UNUSED(argv); + UNUSED(accurate); unsigned int err = 0; int optimize_start = @@ -1703,11 +1710,14 @@ int quicklistTest(int argc, char *argv[]) { printf("Starting optimization offset at: %d\n", optimize_start); int options[] = {0, 1, 2, 3, 4, 5, 6, 10}; + int fills[] = {-5, -4, -3, -2, -1, 0, + 1, 2, 32, 66, 128, 999}; size_t option_count = sizeof(options) / sizeof(*options); + int fill_count = (int)(sizeof(fills) / sizeof(*fills)); long long runtime[option_count]; for (int _i = 0; _i < (int)option_count; _i++) { - printf("Testing Option %d\n", options[_i]); + printf("Testing Compression option %d\n", options[_i]); long long start = mstime(); TEST("create list") { @@ -1732,57 +1742,53 @@ int quicklistTest(int argc, char *argv[]) { quicklistRelease(ql); } - for (int f = optimize_start; f < 32; f++) { - TEST_DESC("add to tail 5x at fill %d at compress %d", f, - options[_i]) { - quicklist *ql = quicklistNew(f, options[_i]); + TEST_DESC("add to tail 5x at compress %d", options[_i]) { + for (int f = 0; f < fill_count; f++) { + quicklist *ql = quicklistNew(fills[f], options[_i]); for (int i = 0; i < 5; i++) quicklistPushTail(ql, genstr("hello", i), 32); if (ql->count != 5) ERROR; - if (f == 32) + if (fills[f] == 32) ql_verify(ql, 1, 5, 5, 5); quicklistRelease(ql); } } - for (int f = optimize_start; f < 32; f++) { - TEST_DESC("add to head 5x at fill %d at compress %d", f, - options[_i]) { - quicklist *ql = quicklistNew(f, options[_i]); + TEST_DESC("add to head 5x at compress %d", options[_i]) { + for (int f = 0; f < fill_count; f++) { + quicklist *ql = quicklistNew(fills[f], options[_i]); for (int i = 0; i < 5; i++) quicklistPushHead(ql, genstr("hello", i), 32); if (ql->count != 5) ERROR; - if (f == 32) + if (fills[f] == 32) ql_verify(ql, 1, 5, 5, 5); quicklistRelease(ql); } } - for (int f = optimize_start; f < 512; f++) { - TEST_DESC("add to tail 500x at fill %d at compress %d", f, - options[_i]) { - quicklist *ql = quicklistNew(f, options[_i]); + TEST_DESC("add to tail 500x at compress %d", options[_i]) { + for (int f = 0; f < fill_count; f++) { + quicklist *ql = quicklistNew(fills[f], options[_i]); for (int i = 0; i < 500; i++) quicklistPushTail(ql, genstr("hello", i), 64); if (ql->count != 500) ERROR; - if (f == 32) + if (fills[f] == 32) ql_verify(ql, 16, 500, 32, 20); quicklistRelease(ql); } } - for (int f = optimize_start; f < 512; f++) { - TEST_DESC("add to head 500x at fill %d at compress %d", f, - options[_i]) { - quicklist *ql = quicklistNew(f, options[_i]); + TEST_DESC("add to head 500x at compress %d", options[_i]) { + for (int f = 0; f < fill_count; f++) { + quicklist *ql = quicklistNew(fills[f], options[_i]); for (int i = 0; i < 500; i++) quicklistPushHead(ql, genstr("hello", i), 32); if (ql->count != 500) ERROR; - if (f == 32) + if (fills[f] == 32) ql_verify(ql, 16, 500, 20, 32); quicklistRelease(ql); } @@ -1795,9 +1801,9 @@ int quicklistTest(int argc, char *argv[]) { quicklistRelease(ql); } - for (int f = optimize_start; f < 32; f++) { - TEST("rotate one val once") { - quicklist *ql = quicklistNew(f, options[_i]); + TEST("rotate one val once") { + for (int f = 0; f < fill_count; f++) { + quicklist *ql = quicklistNew(fills[f], options[_i]); quicklistPushHead(ql, "hello", 6); quicklistRotate(ql); /* Ignore compression verify because ziplist is @@ -1807,10 +1813,9 @@ int quicklistTest(int argc, char *argv[]) { } } - for (int f = optimize_start; f < 3; f++) { - TEST_DESC("rotate 500 val 5000 times at fill %d at compress %d", f, - options[_i]) { - quicklist *ql = quicklistNew(f, options[_i]); + TEST_DESC("rotate 500 val 5000 times at compress %d", options[_i]) { + for (int f = 0; f < fill_count; f++) { + quicklist *ql = quicklistNew(fills[f], options[_i]); quicklistPushHead(ql, "900", 3); quicklistPushHead(ql, "7000", 4); quicklistPushHead(ql, "-1200", 5); @@ -1822,11 +1827,11 @@ int quicklistTest(int argc, char *argv[]) { ql_info(ql); quicklistRotate(ql); } - if (f == 1) + if (fills[f] == 1) ql_verify(ql, 504, 504, 1, 1); - else if (f == 2) + else if (fills[f] == 2) ql_verify(ql, 252, 504, 2, 2); - else if (f == 32) + else if (fills[f] == 32) ql_verify(ql, 16, 504, 32, 24); quicklistRelease(ql); } @@ -2003,11 +2008,10 @@ int quicklistTest(int argc, char *argv[]) { quicklistRelease(ql); } - for (int f = optimize_start; f < 12; f++) { - TEST_DESC("insert once in elements while iterating at fill %d at " - "compress %d\n", - f, options[_i]) { - quicklist *ql = quicklistNew(f, options[_i]); + TEST_DESC("insert once in elements while iterating at compress %d", + options[_i]) { + for (int f = 0; f < fill_count; f++) { + quicklist *ql = quicklistNew(fills[f], options[_i]); quicklistPushTail(ql, "abc", 3); quicklistSetFill(ql, 1); quicklistPushTail(ql, "def", 3); /* force to unique node */ @@ -2059,12 +2063,10 @@ int quicklistTest(int argc, char *argv[]) { } } - for (int f = optimize_start; f < 1024; f++) { - TEST_DESC( - "insert [before] 250 new in middle of 500 elements at fill" - " %d at compress %d", - f, options[_i]) { - quicklist *ql = quicklistNew(f, options[_i]); + TEST_DESC("insert [before] 250 new in middle of 500 elements at compress %d", + options[_i]) { + for (int f = 0; f < fill_count; f++) { + quicklist *ql = quicklistNew(fills[f], options[_i]); for (int i = 0; i < 500; i++) quicklistPushTail(ql, genstr("hello", i), 32); for (int i = 0; i < 250; i++) { @@ -2072,17 +2074,16 @@ int quicklistTest(int argc, char *argv[]) { quicklistIndex(ql, 250, &entry); quicklistInsertBefore(ql, &entry, genstr("abc", i), 32); } - if (f == 32) + if (fills[f] == 32) ql_verify(ql, 25, 750, 32, 20); quicklistRelease(ql); } } - for (int f = optimize_start; f < 1024; f++) { - TEST_DESC("insert [after] 250 new in middle of 500 elements at " - "fill %d at compress %d", - f, options[_i]) { - quicklist *ql = quicklistNew(f, options[_i]); + TEST_DESC("insert [after] 250 new in middle of 500 elements at compress %d", + options[_i]) { + for (int f = 0; f < fill_count; f++) { + quicklist *ql = quicklistNew(fills[f], options[_i]); for (int i = 0; i < 500; i++) quicklistPushHead(ql, genstr("hello", i), 32); for (int i = 0; i < 250; i++) { @@ -2094,7 +2095,7 @@ int quicklistTest(int argc, char *argv[]) { if (ql->count != 750) ERR("List size not 750, but rather %ld", ql->count); - if (f == 32) + if (fills[f] == 32) ql_verify(ql, 26, 750, 20, 32); quicklistRelease(ql); } @@ -2132,70 +2133,58 @@ int quicklistTest(int argc, char *argv[]) { quicklistRelease(copy); } - for (int f = optimize_start; f < 512; f++) { + for (int f = 0; f < fill_count; f++) { TEST_DESC("index 1,200 from 500 list at fill %d at compress %d", f, options[_i]) { - quicklist *ql = quicklistNew(f, options[_i]); + quicklist *ql = quicklistNew(fills[f], options[_i]); for (int i = 0; i < 500; i++) quicklistPushTail(ql, genstr("hello", i + 1), 32); quicklistEntry entry; quicklistIndex(ql, 1, &entry); - if (!strcmp((char *)entry.value, "hello2")) - OK; - else + if (strcmp((char *)entry.value, "hello2") != 0) ERR("Value: %s", entry.value); quicklistIndex(ql, 200, &entry); - if (!strcmp((char *)entry.value, "hello201")) - OK; - else + if (strcmp((char *)entry.value, "hello201") != 0) ERR("Value: %s", entry.value); quicklistRelease(ql); } - TEST_DESC("index -1,-2 from 500 list at fill %d at compress %d", f, - options[_i]) { - quicklist *ql = quicklistNew(f, options[_i]); + TEST_DESC("index -1,-2 from 500 list at fill %d at compress %d", + fills[f], options[_i]) { + quicklist *ql = quicklistNew(fills[f], options[_i]); for (int i = 0; i < 500; i++) quicklistPushTail(ql, genstr("hello", i + 1), 32); quicklistEntry entry; quicklistIndex(ql, -1, &entry); - if (!strcmp((char *)entry.value, "hello500")) - OK; - else + if (strcmp((char *)entry.value, "hello500") != 0) ERR("Value: %s", entry.value); quicklistIndex(ql, -2, &entry); - if (!strcmp((char *)entry.value, "hello499")) - OK; - else + if (strcmp((char *)entry.value, "hello499") != 0) ERR("Value: %s", entry.value); quicklistRelease(ql); } - TEST_DESC("index -100 from 500 list at fill %d at compress %d", f, - options[_i]) { - quicklist *ql = quicklistNew(f, options[_i]); + TEST_DESC("index -100 from 500 list at fill %d at compress %d", + fills[f], options[_i]) { + quicklist *ql = quicklistNew(fills[f], options[_i]); for (int i = 0; i < 500; i++) quicklistPushTail(ql, genstr("hello", i + 1), 32); quicklistEntry entry; quicklistIndex(ql, -100, &entry); - if (!strcmp((char *)entry.value, "hello401")) - OK; - else + if (strcmp((char *)entry.value, "hello401") != 0) ERR("Value: %s", entry.value); quicklistRelease(ql); } TEST_DESC("index too big +1 from 50 list at fill %d at compress %d", - f, options[_i]) { - quicklist *ql = quicklistNew(f, options[_i]); + fills[f], options[_i]) { + quicklist *ql = quicklistNew(fills[f], options[_i]); for (int i = 0; i < 50; i++) quicklistPushTail(ql, genstr("hello", i + 1), 32); quicklistEntry entry; if (quicklistIndex(ql, 50, &entry)) ERR("Index found at 50 with 50 list: %.*s", entry.sz, entry.value); - else - OK; quicklistRelease(ql); } } @@ -2367,12 +2356,11 @@ int quicklistTest(int argc, char *argv[]) { quicklistReplaceAtIndex(ql, 1, "foo", 3); quicklistReplaceAtIndex(ql, -1, "bar", 3); quicklistRelease(ql); - OK; } - for (int f = optimize_start; f < 16; f++) { - TEST_DESC("lrem test at fill %d at compress %d", f, options[_i]) { - quicklist *ql = quicklistNew(f, options[_i]); + TEST_DESC("lrem test at compress %d", options[_i]) { + for (int f = 0; f < fill_count; f++) { + quicklist *ql = quicklistNew(fills[f], options[_i]); char *words[] = {"abc", "foo", "bar", "foobar", "foobared", "zap", "bar", "test", "foo"}; char *result[] = {"abc", "foo", "foobar", "foobared", @@ -2397,14 +2385,12 @@ int quicklistTest(int argc, char *argv[]) { /* check result of lrem 0 bar */ iter = quicklistGetIterator(ql, AL_START_HEAD); i = 0; - int ok = 1; while (quicklistNext(iter, &entry)) { /* Result must be: abc, foo, foobar, foobared, zap, test, * foo */ if (strncmp((char *)entry.value, result[i], entry.sz)) { ERR("No match at position %d, got %.*s instead of %s", i, entry.sz, entry.value, result[i]); - ok = 0; } i++; } @@ -2441,23 +2427,18 @@ int quicklistTest(int argc, char *argv[]) { entry.sz)) { ERR("No match at position %d, got %.*s instead of %s", i, entry.sz, entry.value, resultB[resB - 1 - i]); - ok = 0; } i++; } quicklistReleaseIterator(iter); - /* final result of all tests */ - if (ok) - OK; quicklistRelease(ql); } } - for (int f = optimize_start; f < 16; f++) { - TEST_DESC("iterate reverse + delete at fill %d at compress %d", f, - options[_i]) { - quicklist *ql = quicklistNew(f, options[_i]); + TEST_DESC("iterate reverse + delete at compress %d", options[_i]) { + for (int f = 0; f < fill_count; f++) { + quicklist *ql = quicklistNew(fills[f], options[_i]); quicklistPushTail(ql, "abc", 3); quicklistPushTail(ql, "def", 3); quicklistPushTail(ql, "hij", 3); @@ -2494,10 +2475,9 @@ int quicklistTest(int argc, char *argv[]) { } } - for (int f = optimize_start; f < 800; f++) { - TEST_DESC("iterator at index test at fill %d at compress %d", f, - options[_i]) { - quicklist *ql = quicklistNew(f, options[_i]); + TEST_DESC("iterator at index test at compress %d", options[_i]) { + for (int f = 0; f < fill_count; f++) { + quicklist *ql = quicklistNew(fills[f], options[_i]); char num[32]; long long nums[5000]; for (int i = 0; i < 760; i++) { @@ -2521,10 +2501,9 @@ int quicklistTest(int argc, char *argv[]) { } } - for (int f = optimize_start; f < 40; f++) { - TEST_DESC("ltrim test A at fill %d at compress %d", f, - options[_i]) { - quicklist *ql = quicklistNew(f, options[_i]); + TEST_DESC("ltrim test A at compress %d", options[_i]) { + for (int f = 0; f < fill_count; f++) { + quicklist *ql = quicklistNew(fills[f], options[_i]); char num[32]; long long nums[5000]; for (int i = 0; i < 32; i++) { @@ -2532,7 +2511,7 @@ int quicklistTest(int argc, char *argv[]) { int sz = ll2string(num, sizeof(num), nums[i]); quicklistPushTail(ql, num, sz); } - if (f == 32) + if (fills[f] == 32) ql_verify(ql, 1, 32, 32, 32); /* ltrim 25 53 (keep [25,32] inclusive = 7 remaining) */ quicklistDelRange(ql, 0, 25); @@ -2545,18 +2524,17 @@ int quicklistTest(int argc, char *argv[]) { "%lld", entry.longval, nums[25 + i]); } - if (f == 32) + if (fills[f] == 32) ql_verify(ql, 1, 7, 7, 7); quicklistRelease(ql); } } - for (int f = optimize_start; f < 40; f++) { - TEST_DESC("ltrim test B at fill %d at compress %d", f, - options[_i]) { + TEST_DESC("ltrim test B at compress %d", options[_i]) { + for (int f = 0; f < fill_count; f++) { /* Force-disable compression because our 33 sequential * integers don't compress and the check always fails. */ - quicklist *ql = quicklistNew(f, QUICKLIST_NOCOMPRESS); + quicklist *ql = quicklistNew(fills[f], QUICKLIST_NOCOMPRESS); char num[32]; long long nums[5000]; for (int i = 0; i < 33; i++) { @@ -2564,24 +2542,20 @@ int quicklistTest(int argc, char *argv[]) { int sz = ll2string(num, sizeof(num), nums[i]); quicklistPushTail(ql, num, sz); } - if (f == 32) + if (fills[f] == 32) ql_verify(ql, 2, 33, 32, 1); /* ltrim 5 16 (keep [5,16] inclusive = 12 remaining) */ quicklistDelRange(ql, 0, 5); quicklistDelRange(ql, -16, 16); - if (f == 32) + if (fills[f] == 32) ql_verify(ql, 1, 12, 12, 12); quicklistEntry entry; quicklistIndex(ql, 0, &entry); if (entry.longval != 5) ERR("A: longval not 5, but %lld", entry.longval); - else - OK; quicklistIndex(ql, -1, &entry); if (entry.longval != 16) ERR("B! got instead: %lld", entry.longval); - else - OK; quicklistPushTail(ql, "bobobob", 7); quicklistIndex(ql, -1, &entry); if (strncmp((char *)entry.value, "bobobob", 7)) @@ -2598,10 +2572,9 @@ int quicklistTest(int argc, char *argv[]) { } } - for (int f = optimize_start; f < 40; f++) { - TEST_DESC("ltrim test C at fill %d at compress %d", f, - options[_i]) { - quicklist *ql = quicklistNew(f, options[_i]); + TEST_DESC("ltrim test C at compress %d", options[_i]) { + for (int f = 0; f < fill_count; f++) { + quicklist *ql = quicklistNew(fills[f], options[_i]); char num[32]; long long nums[5000]; for (int i = 0; i < 33; i++) { @@ -2609,28 +2582,25 @@ int quicklistTest(int argc, char *argv[]) { int sz = ll2string(num, sizeof(num), nums[i]); quicklistPushTail(ql, num, sz); } - if (f == 32) + if (fills[f] == 32) ql_verify(ql, 2, 33, 32, 1); /* ltrim 3 3 (keep [3,3] inclusive = 1 remaining) */ quicklistDelRange(ql, 0, 3); quicklistDelRange(ql, -29, 4000); /* make sure not loop forever */ - if (f == 32) + if (fills[f] == 32) ql_verify(ql, 1, 1, 1, 1); quicklistEntry entry; quicklistIndex(ql, 0, &entry); if (entry.longval != -5157318210846258173) ERROR; - else - OK; quicklistRelease(ql); } } - for (int f = optimize_start; f < 40; f++) { - TEST_DESC("ltrim test D at fill %d at compress %d", f, - options[_i]) { - quicklist *ql = quicklistNew(f, options[_i]); + TEST_DESC("ltrim test D at compress %d", options[_i]) { + for (int f = 0; f < fill_count; f++) { + quicklist *ql = quicklistNew(fills[f], options[_i]); char num[32]; long long nums[5000]; for (int i = 0; i < 33; i++) { @@ -2638,7 +2608,7 @@ int quicklistTest(int argc, char *argv[]) { int sz = ll2string(num, sizeof(num), nums[i]); quicklistPushTail(ql, num, sz); } - if (f == 32) + if (fills[f] == 32) ql_verify(ql, 2, 33, 32, 1); quicklistDelRange(ql, -12, 3); if (ql->count != 30) @@ -2648,9 +2618,8 @@ int quicklistTest(int argc, char *argv[]) { } } - for (int f = optimize_start; f < 72; f++) { - TEST_DESC("create quicklist from ziplist at fill %d at compress %d", - f, options[_i]) { + TEST_DESC("create quicklist from ziplist at compress %d", options[_i]) { + for (int f = 0; f < fill_count; f++) { unsigned char *zl = ziplistNew(); long long nums[64]; char num[64]; @@ -2664,12 +2633,12 @@ int quicklistTest(int argc, char *argv[]) { zl = ziplistPush(zl, (unsigned char *)genstr("hello", i), 32, ZIPLIST_TAIL); } - quicklist *ql = quicklistCreateFromZiplist(f, options[_i], zl); - if (f == 1) + quicklist *ql = quicklistCreateFromZiplist(fills[f], options[_i], zl); + if (fills[f] == 1) ql_verify(ql, 66, 66, 1, 1); - else if (f == 32) + else if (fills[f] == 32) ql_verify(ql, 3, 66, 32, 2); - else if (f == 66) + else if (fills[f] == 66) ql_verify(ql, 1, 66, 66, 66); quicklistRelease(ql); } @@ -2682,45 +2651,56 @@ int quicklistTest(int argc, char *argv[]) { /* Run a longer test of compression depth outside of primary test loop. */ int list_sizes[] = {250, 251, 500, 999, 1000}; long long start = mstime(); - for (int list = 0; list < (int)(sizeof(list_sizes) / sizeof(*list_sizes)); - list++) { - for (int f = optimize_start; f < 128; f++) { - for (int depth = 1; depth < 40; depth++) { - /* skip over many redundant test cases */ - TEST_DESC("verify specific compression of interior nodes with " - "%d list " - "at fill %d at compress %d", - list_sizes[list], f, depth) { - quicklist *ql = quicklistNew(f, depth); + int list_count = accurate ? (int)(sizeof(list_sizes) / sizeof(*list_sizes)) : 1; + for (int list = 0; list < list_count; list++) { + TEST_DESC("verify specific compression of interior nodes with %d list ", + list_sizes[list]) { + for (int f = 0; f < fill_count; f++) { + for (int depth = 1; depth < 40; depth++) { + /* skip over many redundant test cases */ + quicklist *ql = quicklistNew(fills[f], depth); for (int i = 0; i < list_sizes[list]; i++) { quicklistPushTail(ql, genstr("hello TAIL", i + 1), 64); quicklistPushHead(ql, genstr("hello HEAD", i + 1), 64); } - quicklistNode *node = ql->head; - unsigned int low_raw = ql->compress; - unsigned int high_raw = ql->len - ql->compress; - - for (unsigned int at = 0; at < ql->len; - at++, node = node->next) { - if (at < low_raw || at >= high_raw) { - if (node->encoding != QUICKLIST_NODE_ENCODING_RAW) { - ERR("Incorrect compression: node %d is " - "compressed at depth %d ((%u, %u); total " - "nodes: %lu; size: %u)", - at, depth, low_raw, high_raw, ql->len, - node->sz); + for (int step = 0; step < 2; step++) { + /* test remove node */ + if (step == 1) { + for (int i = 0; i < list_sizes[list] / 2; i++) { + unsigned char *data; + quicklistPop(ql, QUICKLIST_HEAD, &data, NULL, NULL); + zfree(data); + quicklistPop(ql, QUICKLIST_TAIL, &data, NULL, NULL); + zfree(data); } - } else { - if (node->encoding != QUICKLIST_NODE_ENCODING_LZF) { - ERR("Incorrect non-compression: node %d is NOT " - "compressed at depth %d ((%u, %u); total " - "nodes: %lu; size: %u; attempted: %d)", - at, depth, low_raw, high_raw, ql->len, - node->sz, node->attempted_compress); + } + quicklistNode *node = ql->head; + unsigned int low_raw = ql->compress; + unsigned int high_raw = ql->len - ql->compress; + + for (unsigned int at = 0; at < ql->len; + at++, node = node->next) { + if (at < low_raw || at >= high_raw) { + if (node->encoding != QUICKLIST_NODE_ENCODING_RAW) { + ERR("Incorrect compression: node %d is " + "compressed at depth %d ((%u, %u); total " + "nodes: %lu; size: %u)", + at, depth, low_raw, high_raw, ql->len, + node->sz); + } + } else { + if (node->encoding != QUICKLIST_NODE_ENCODING_LZF) { + ERR("Incorrect non-compression: node %d is NOT " + "compressed at depth %d ((%u, %u); total " + "nodes: %lu; size: %u; attempted: %d)", + at, depth, low_raw, high_raw, ql->len, + node->sz, node->attempted_compress); + } } } } + quicklistRelease(ql); } } diff --git a/src/quicklist.h b/src/quicklist.h index fd9878af0..c9f493d80 100644 --- a/src/quicklist.h +++ b/src/quicklist.h @@ -199,7 +199,7 @@ quicklistNode *quicklistBookmarkFind(quicklist *ql, const char *name); void quicklistBookmarksClear(quicklist *ql); #ifdef REDIS_TEST -int quicklistTest(int argc, char *argv[]); +int quicklistTest(int argc, char *argv[], int accurate); #endif /* Directions for iterators */ diff --git a/src/rdb.c b/src/rdb.c index 630417302..6f2f5165e 100644 --- a/src/rdb.c +++ b/src/rdb.c @@ -1073,8 +1073,7 @@ size_t rdbSavedObjectLen(robj *o, robj *key) { /* Save a key-value pair, with expire time, type, key, value. * On error -1 is returned. - * On success if the key was actually saved 1 is returned, otherwise 0 - * is returned (the key was already expired). */ + * On success if the key was actually saved 1 is returned. */ int rdbSaveKeyValuePair(rio *rdb, robj *key, robj *val, long long expiretime) { int savelru = server.maxmemory_policy & MAXMEMORY_FLAG_LRU; int savelfu = server.maxmemory_policy & MAXMEMORY_FLAG_LFU; @@ -2176,16 +2175,17 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key) { return NULL; } moduleType *mt = moduleTypeLookupModuleByID(moduleid); - char name[10]; if (rdbCheckMode && rdbtype == RDB_TYPE_MODULE_2) { + char name[10]; moduleTypeNameByID(name,moduleid); return rdbLoadCheckModuleValue(rdb,name); } if (mt == NULL) { + char name[10]; moduleTypeNameByID(name,moduleid); - rdbReportCorruptRDB("The RDB file contains module data I can't load: no matching module '%s'", name); + rdbReportCorruptRDB("The RDB file contains module data I can't load: no matching module type '%s'", name); return NULL; } RedisModuleIO io; @@ -2212,7 +2212,8 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key) { return NULL; } if (eof != RDB_MODULE_OPCODE_EOF) { - rdbReportCorruptRDB("The RDB file contains module data for the module '%s' that is not terminated by the proper module value EOF marker", name); + rdbReportCorruptRDB("The RDB file contains module data for the module '%s' that is not terminated by " + "the proper module value EOF marker", moduleTypeModuleName(mt)); if (ptr) { o = createModuleObject(mt,ptr); /* creating just in order to easily destroy */ decrRefCount(o); @@ -2222,8 +2223,9 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key) { } if (ptr == NULL) { - moduleTypeNameByID(name,moduleid); - rdbReportCorruptRDB("The RDB file contains module data for the module type '%s', that the responsible module is not able to load. Check for modules log above for additional clues.", name); + rdbReportCorruptRDB("The RDB file contains module data for the module type '%s', that the responsible " + "module is not able to load. Check for modules log above for additional clues.", + moduleTypeModuleName(mt)); return NULL; } o = createModuleObject(mt,ptr); @@ -2729,7 +2731,7 @@ void backgroundSaveDoneHandler(int exitcode, int bysignal) { * the cleanup needed. */ void killRDBChild(void) { kill(server.child_pid, SIGUSR1); - /* Because we are not using here wait4 (like we have in killAppendOnlyChild + /* Because we are not using here waitpid (like we have in killAppendOnlyChild * and TerminateModuleForkChild), all the cleanup operations is done by * checkChildrenDone, that later will find that the process killed. * This includes: diff --git a/src/redis-benchmark.c b/src/redis-benchmark.c index 186739766..351335862 100644 --- a/src/redis-benchmark.c +++ b/src/redis-benchmark.c @@ -102,7 +102,6 @@ static struct config { int showerrors; long long start; long long totlatency; - long long *latency; const char *title; list *clients; int quiet; @@ -1656,7 +1655,10 @@ int showThroughput(struct aeEventLoop *eventLoop, long long id, void *clientData const float instantaneous_rps = (float)(requests_finished-previous_requests_finished)/instantaneous_dt; config.previous_tick = current_tick; atomicSet(config.previous_requests_finished,requests_finished); - config.last_printed_bytes = printf("%s: rps=%.1f (overall: %.1f) avg_msec=%.3f (overall: %.3f)\r", config.title, instantaneous_rps, rps, hdr_mean(config.current_sec_latency_histogram)/1000.0f, hdr_mean(config.latency_histogram)/1000.0f); + int printed_bytes = printf("%s: rps=%.1f (overall: %.1f) avg_msec=%.3f (overall: %.3f)\r", config.title, instantaneous_rps, rps, hdr_mean(config.current_sec_latency_histogram)/1000.0f, hdr_mean(config.latency_histogram)/1000.0f); + if (printed_bytes > config.last_printed_bytes){ + config.last_printed_bytes = printed_bytes; + } hdr_reset(config.current_sec_latency_histogram); fflush(stdout); return 250; /* every 250ms */ diff --git a/src/redis-check-rdb.c b/src/redis-check-rdb.c index 335e35189..4f451969a 100644 --- a/src/redis-check-rdb.c +++ b/src/redis-check-rdb.c @@ -192,6 +192,7 @@ int redis_check_rdb(char *rdbfilename, FILE *fp) { int closefile = (fp == NULL); if (fp == NULL && (fp = fopen(rdbfilename,"r")) == NULL) return 1; + startLoadingFile(fp, rdbfilename, RDBFLAGS_NONE); rioInitWithFile(&rdb,fp); rdbstate.rio = &rdb; rdb.update_cksum = rdbLoadProgressCallback; @@ -208,7 +209,6 @@ int redis_check_rdb(char *rdbfilename, FILE *fp) { } expiretime = -1; - startLoadingFile(fp, rdbfilename, RDBFLAGS_NONE); while(1) { robj *key, *val; diff --git a/src/redis-cli.c b/src/redis-cli.c index 63d31f79a..7e1fe3934 100644 --- a/src/redis-cli.c +++ b/src/redis-cli.c @@ -227,7 +227,7 @@ static struct config { int scan_mode; int intrinsic_latency_mode; int intrinsic_latency_duration; - char *pattern; + sds pattern; char *rdb_filename; int bigkeys; int memkeys; @@ -237,6 +237,7 @@ static struct config { char *auth; int askpass; char *user; + int quoted_input; /* Force input args to be treated as quoted strings */ int output; /* output mode, see OUTPUT_* defines */ int push_output; /* Should we display spontaneous PUSH replies */ sds mb_delim; @@ -405,15 +406,17 @@ static void parseRedisUri(const char *uri) { if (!strncasecmp(tlsscheme, curr, strlen(tlsscheme))) { #ifdef USE_OPENSSL config.tls = 1; + curr += strlen(tlsscheme); #else fprintf(stderr,"rediss:// is only supported when redis-cli is compiled with OpenSSL\n"); exit(1); #endif - } else if (strncasecmp(scheme, curr, strlen(scheme))) { + } else if (!strncasecmp(scheme, curr, strlen(scheme))) { + curr += strlen(scheme); + } else { fprintf(stderr,"Invalid URI scheme\n"); exit(1); } - curr += strlen(scheme); if (curr == end) return; /* Extract user info. */ @@ -763,6 +766,23 @@ static void freeHintsCallback(void *ptr) { * Networking / parsing *--------------------------------------------------------------------------- */ +/* Unquote a null-terminated string and return it as a binary-safe sds. */ +static sds unquoteCString(char *str) { + int count; + sds *unquoted = sdssplitargs(str, &count); + sds res = NULL; + + if (unquoted && count == 1) { + res = unquoted[0]; + unquoted[0] = NULL; + } + + if (unquoted) + sdsfreesplitres(unquoted, count); + + return res; +} + /* Send AUTH command to the server */ static int cliAuth(redisContext *ctx, char *user, char *auth) { redisReply *reply; @@ -1533,6 +1553,8 @@ static int parseOptions(int argc, char **argv) { config.output = OUTPUT_RAW; } else if (!strcmp(argv[i],"--no-raw")) { config.output = OUTPUT_STANDARD; + } else if (!strcmp(argv[i],"--quoted-input")) { + config.quoted_input = 1; } else if (!strcmp(argv[i],"--csv")) { config.output = OUTPUT_CSV; } else if (!strcmp(argv[i],"--latency")) { @@ -1557,7 +1579,15 @@ static int parseOptions(int argc, char **argv) { } else if (!strcmp(argv[i],"--scan")) { config.scan_mode = 1; } else if (!strcmp(argv[i],"--pattern") && !lastarg) { - config.pattern = argv[++i]; + sdsfree(config.pattern); + config.pattern = sdsnew(argv[++i]); + } else if (!strcmp(argv[i],"--quoted-pattern") && !lastarg) { + sdsfree(config.pattern); + config.pattern = unquoteCString(argv[++i]); + if (!config.pattern) { + fprintf(stderr,"Invalid quoted string specified for --quoted-pattern.\n"); + exit(1); + } } else if (!strcmp(argv[i],"--intrinsic-latency") && !lastarg) { config.intrinsic_latency_mode = 1; config.intrinsic_latency_duration = atoi(argv[++i]); @@ -1841,6 +1871,7 @@ static void usage(void) { " --raw Use raw formatting for replies (default when STDOUT is\n" " not a tty).\n" " --no-raw Force formatted output even when STDOUT is not a tty.\n" +" --quoted-input Force input to be handled as quoted strings.\n" " --csv Output in CSV format.\n" " --show-pushes Whether to print RESP3 PUSH messages. Enabled by default when\n" " STDOUT is a tty but can be overriden with --show-pushes no.\n" @@ -1876,6 +1907,8 @@ static void usage(void) { " --scan List all keys using the SCAN command.\n" " --pattern Keys pattern when using the --scan, --bigkeys or --hotkeys\n" " options (default: *).\n" +" --quoted-pattern Same as --pattern, but the specified string can be\n" +" quoted, in order to pass an otherwise non binary-safe string.\n" " --intrinsic-latency Run a test to measure intrinsic system latency.\n" " The test will run for the specified amount of seconds.\n" " --eval Send an EVAL command using the Lua script at .\n" @@ -1901,6 +1934,7 @@ static void usage(void) { " redis-cli get mypasswd\n" " redis-cli -r 100 lpush mylist x\n" " redis-cli -r 100 -i 1 info | grep used_memory_human:\n" +" redis-cli --quoted-input set '\"null-\\x00-separated\"' value\n" " redis-cli --eval myscript.lua key1 key2 , arg1 arg2 arg3\n" " redis-cli --scan --pattern '*:12345*'\n" "\n" @@ -1930,22 +1964,28 @@ static int confirmWithYes(char *msg, int ignore_force) { return (nread != 0 && !strcmp("yes", buf)); } -/* Turn the plain C strings into Sds strings */ -static char **convertToSds(int count, char** args) { - int j; - char **sds = zmalloc(sizeof(char*)*count); +/* Create an sds array from argv, either as-is or by dequoting every + * element. When quoted is non-zero, may return a NULL to indicate an + * invalid quoted string. + */ +static sds *getSdsArrayFromArgv(int argc, char **argv, int quoted) { + sds *res = sds_malloc(sizeof(sds) * argc); - for(j = 0; j < count; j++) - sds[j] = sdsnew(args[j]); + for (int j = 0; j < argc; j++) { + if (quoted) { + sds unquoted = unquoteCString(argv[j]); + if (!unquoted) { + while (--j >= 0) sdsfree(res[j]); + sds_free(res); + return NULL; + } + res[j] = unquoted; + } else { + res[j] = sdsnew(argv[j]); + } + } - return sds; -} - -static void freeConvertedSds(int count, char **sds) { - int j; - for (j = 0; j < count; j++) - sdsfree(sds[j]); - zfree(sds); + return res; } static int issueCommandRepeat(int argc, char **argv, long repeat) { @@ -2178,17 +2218,19 @@ static void repl(void) { static int noninteractive(int argc, char **argv) { int retval = 0; - - argv = convertToSds(argc, argv); - if (config.stdinarg) { - argv = zrealloc(argv, (argc+1)*sizeof(char*)); - argv[argc] = readArgFromStdin(); - retval = issueCommand(argc+1, argv); - sdsfree(argv[argc]); - } else { - retval = issueCommand(argc, argv); + sds *sds_args = getSdsArrayFromArgv(argc, argv, config.quoted_input); + if (!sds_args) { + printf("Invalid quoted string\n"); + return 1; } - freeConvertedSds(argc, argv); + if (config.stdinarg) { + sds_args = sds_realloc(sds_args, (argc + 1) * sizeof(sds)); + sds_args[argc] = readArgFromStdin(); + argc++; + } + + retval = issueCommand(argc, sds_args); + sdsfreesplitres(sds_args, argc); return retval; } @@ -2913,8 +2955,12 @@ static int clusterManagerGetAntiAffinityScore(clusterManagerNodeArray *ipnodes, else types = sdsempty(); /* Master type 'm' is always set as the first character of the * types string. */ - if (!node->replicate) types = sdscatprintf(types, "m%s", types); - else types = sdscat(types, "s"); + if (node->replicate) types = sdscat(types, "s"); + else { + sds s = sdscatsds(sdsnew("m"), types); + sdsfree(types); + types = s; + } dictReplace(related, key, types); } /* Now it's trivial to check, for each related group having the @@ -7156,7 +7202,10 @@ static void getRDB(clusterManagerNode *node) { redisFree(s); /* Close the connection ASAP as fsync() may take time. */ if (node) node->context = NULL; - fsync(fd); + if (fsync(fd) == -1) { + fprintf(stderr,"Fail to fsync '%s': %s\n", filename, strerror(errno)); + exit(1); + } close(fd); if (node) { sdsfree(filename); @@ -7332,8 +7381,8 @@ static redisReply *sendScan(unsigned long long *it) { redisReply *reply; if (config.pattern) - reply = redisCommand(context,"SCAN %llu MATCH %s", - *it,config.pattern); + reply = redisCommand(context, "SCAN %llu MATCH %b", + *it, config.pattern, sdslen(config.pattern)); else reply = redisCommand(context,"SCAN %llu",*it); @@ -7368,8 +7417,14 @@ static int getDbSize(void) { reply = redisCommand(context, "DBSIZE"); - if(reply == NULL || reply->type != REDIS_REPLY_INTEGER) { - fprintf(stderr, "Couldn't determine DBSIZE!\n"); + if (reply == NULL) { + fprintf(stderr, "\nI/O error\n"); + exit(1); + } else if (reply->type == REDIS_REPLY_ERROR) { + fprintf(stderr, "Couldn't determine DBSIZE: %s\n", reply->str); + exit(1); + } else if (reply->type != REDIS_REPLY_INTEGER) { + fprintf(stderr, "Non INTEGER response from DBSIZE!\n"); exit(1); } @@ -7945,23 +8000,16 @@ static void scanMode(void) { unsigned long long cur = 0; do { - if (config.pattern) - reply = redisCommand(context,"SCAN %llu MATCH %s", - cur,config.pattern); - else - reply = redisCommand(context,"SCAN %llu",cur); - if (reply == NULL) { - printf("I/O error\n"); - exit(1); - } else if (reply->type == REDIS_REPLY_ERROR) { - printf("ERROR: %s\n", reply->str); - exit(1); - } else { - unsigned int j; - - cur = strtoull(reply->element[0]->str,NULL,10); - for (j = 0; j < reply->element[1]->elements; j++) + reply = sendScan(&cur); + for (unsigned int j = 0; j < reply->element[1]->elements; j++) { + if (config.output == OUTPUT_STANDARD) { + sds out = sdscatrepr(sdsempty(), reply->element[1]->element[j]->str, + reply->element[1]->element[j]->len); + printf("%s\n", out); + sdsfree(out); + } else { printf("%s\n", reply->element[1]->element[j]->str); + } } freeReplyObject(reply); } while(cur != 0); diff --git a/src/redismodule.h b/src/redismodule.h index ea271b82b..5520ca3cc 100644 --- a/src/redismodule.h +++ b/src/redismodule.h @@ -160,13 +160,14 @@ This flag should not be used directly by the module. #define REDISMODULE_NOTIFY_STREAM (1<<10) /* t */ #define REDISMODULE_NOTIFY_KEY_MISS (1<<11) /* m (Note: This one is excluded from REDISMODULE_NOTIFY_ALL on purpose) */ #define REDISMODULE_NOTIFY_LOADED (1<<12) /* module only key space notification, indicate a key loaded from rdb */ +#define REDISMODULE_NOTIFY_MODULE (1<<13) /* d, module key space notification */ /* Next notification flag, must be updated when adding new flags above! This flag should not be used directly by the module. * Use RedisModule_GetKeyspaceNotificationFlagsAll instead. */ -#define _REDISMODULE_NOTIFY_NEXT (1<<13) +#define _REDISMODULE_NOTIFY_NEXT (1<<14) -#define REDISMODULE_NOTIFY_ALL (REDISMODULE_NOTIFY_GENERIC | REDISMODULE_NOTIFY_STRING | REDISMODULE_NOTIFY_LIST | REDISMODULE_NOTIFY_SET | REDISMODULE_NOTIFY_HASH | REDISMODULE_NOTIFY_ZSET | REDISMODULE_NOTIFY_EXPIRED | REDISMODULE_NOTIFY_EVICTED | REDISMODULE_NOTIFY_STREAM) /* A */ +#define REDISMODULE_NOTIFY_ALL (REDISMODULE_NOTIFY_GENERIC | REDISMODULE_NOTIFY_STRING | REDISMODULE_NOTIFY_LIST | REDISMODULE_NOTIFY_SET | REDISMODULE_NOTIFY_HASH | REDISMODULE_NOTIFY_ZSET | REDISMODULE_NOTIFY_EXPIRED | REDISMODULE_NOTIFY_EVICTED | REDISMODULE_NOTIFY_STREAM | REDISMODULE_NOTIFY_MODULE) /* A */ /* A special pointer that we can use between the core and the module to signal * field deletion, and that is impossible to be a valid pointer. */ @@ -193,6 +194,12 @@ This flag should not be used directly by the module. #define REDISMODULE_NOT_USED(V) ((void) V) +/* Logging level strings */ +#define REDISMODULE_LOGLEVEL_DEBUG "debug" +#define REDISMODULE_LOGLEVEL_VERBOSE "verbose" +#define REDISMODULE_LOGLEVEL_NOTICE "notice" +#define REDISMODULE_LOGLEVEL_WARNING "warning" + /* Bit flags for aux_save_triggers and the aux_load and aux_save callbacks */ #define REDISMODULE_AUX_BEFORE_RDB (1<<0) #define REDISMODULE_AUX_AFTER_RDB (1<<1) @@ -635,6 +642,8 @@ REDISMODULE_API char * (*RedisModule_StringDMA)(RedisModuleKey *key, size_t *len REDISMODULE_API int (*RedisModule_StringTruncate)(RedisModuleKey *key, size_t newlen) REDISMODULE_ATTR; REDISMODULE_API mstime_t (*RedisModule_GetExpire)(RedisModuleKey *key) REDISMODULE_ATTR; REDISMODULE_API int (*RedisModule_SetExpire)(RedisModuleKey *key, mstime_t expire) REDISMODULE_ATTR; +REDISMODULE_API mstime_t (*RedisModule_GetAbsExpire)(RedisModuleKey *key) REDISMODULE_ATTR; +REDISMODULE_API int (*RedisModule_SetAbsExpire)(RedisModuleKey *key, mstime_t expire) REDISMODULE_ATTR; REDISMODULE_API void (*RedisModule_ResetDataset)(int restart_aof, int async) REDISMODULE_ATTR; REDISMODULE_API unsigned long long (*RedisModule_DbSize)(RedisModuleCtx *ctx) REDISMODULE_ATTR; REDISMODULE_API RedisModuleString * (*RedisModule_RandomKey)(RedisModuleCtx *ctx) REDISMODULE_ATTR; @@ -835,7 +844,7 @@ REDISMODULE_API int (*RedisModule_DefragCursorSet)(RedisModuleDefragCtx *ctx, un REDISMODULE_API int (*RedisModule_DefragCursorGet)(RedisModuleDefragCtx *ctx, unsigned long *cursor) REDISMODULE_ATTR; #endif -#define RedisModule_IsAOFClient(id) ((id) == CLIENT_ID_AOF) +#define RedisModule_IsAOFClient(id) ((id) == UINT64_MAX) /* This is included inline inside each Redis module. */ static int RedisModule_Init(RedisModuleCtx *ctx, const char *name, int ver, int apiver) REDISMODULE_ATTR_UNUSED; @@ -907,6 +916,8 @@ static int RedisModule_Init(RedisModuleCtx *ctx, const char *name, int ver, int REDISMODULE_GET_API(StringTruncate); REDISMODULE_GET_API(GetExpire); REDISMODULE_GET_API(SetExpire); + REDISMODULE_GET_API(GetAbsExpire); + REDISMODULE_GET_API(SetAbsExpire); REDISMODULE_GET_API(ResetDataset); REDISMODULE_GET_API(DbSize); REDISMODULE_GET_API(RandomKey); diff --git a/src/replication.c b/src/replication.c index eb5fa54c0..8177eb073 100644 --- a/src/replication.c +++ b/src/replication.c @@ -892,17 +892,34 @@ void syncCommand(client *c) { } /* REPLCONF