From b1f2c51037837a4b4ab3175276a1c7a8d5247f9c Mon Sep 17 00:00:00 2001 From: "zhaozhao.zz" Date: Tue, 21 May 2019 11:37:13 +0800 Subject: [PATCH 001/225] Threaded IO: use main thread to handle write work --- src/networking.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/networking.c b/src/networking.c index a558ae91a..900e8cf87 100644 --- a/src/networking.c +++ b/src/networking.c @@ -2656,7 +2656,7 @@ pthread_mutex_t io_threads_mutex[IO_THREADS_MAX_NUM]; _Atomic unsigned long io_threads_pending[IO_THREADS_MAX_NUM]; int io_threads_active; /* Are the threads currently spinning waiting I/O? */ int io_threads_op; /* IO_THREADS_OP_WRITE or IO_THREADS_OP_READ. */ -list *io_threads_list[IO_THREADS_MAX_NUM]; +list *io_threads_list[IO_THREADS_MAX_NUM+1]; void *IOThreadMain(void *myid) { /* The ID is the thread number (from 0 to server.iothreads_num-1), and is @@ -2729,6 +2729,7 @@ void initThreadedIO(void) { } io_threads[i] = tid; } + io_threads_list[server.io_threads_num] = listCreate(); } void startThreadedIO(void) { @@ -2800,7 +2801,7 @@ int handleClientsWithPendingWritesUsingThreads(void) { while((ln = listNext(&li))) { client *c = listNodeValue(ln); c->flags &= ~CLIENT_PENDING_WRITE; - int target_id = item_id % server.io_threads_num; + int target_id = item_id % (server.io_threads_num+1); listAddNodeTail(io_threads_list[target_id],c); item_id++; } @@ -2813,6 +2814,13 @@ int handleClientsWithPendingWritesUsingThreads(void) { io_threads_pending[j] = count; } + listRewind(io_threads_list[server.io_threads_num],&li); + while((ln = listNext(&li))) { + client *c = listNodeValue(ln); + writeToClient(c->fd,c,0); + } + listEmpty(io_threads_list[server.io_threads_num]); + /* Wait for all threads to end their work. */ while(1) { unsigned long pending = 0; From b3ff8a4b6d849fc7d9c8737ba9c74c1872142737 Mon Sep 17 00:00:00 2001 From: "zhaozhao.zz" Date: Tue, 21 May 2019 11:42:10 +0800 Subject: [PATCH 002/225] Threaded IO: use main thread to handle read work --- src/networking.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/networking.c b/src/networking.c index 900e8cf87..f1a6b9910 100644 --- a/src/networking.c +++ b/src/networking.c @@ -2885,7 +2885,7 @@ int handleClientsWithPendingReadsUsingThreads(void) { int item_id = 0; while((ln = listNext(&li))) { client *c = listNodeValue(ln); - int target_id = item_id % server.io_threads_num; + int target_id = item_id % (server.io_threads_num+1); listAddNodeTail(io_threads_list[target_id],c); item_id++; } @@ -2898,6 +2898,13 @@ int handleClientsWithPendingReadsUsingThreads(void) { io_threads_pending[j] = count; } + listRewind(io_threads_list[server.io_threads_num],&li); + while((ln = listNext(&li))) { + client *c = listNodeValue(ln); + readQueryFromClient(NULL,c->fd,c,0); + } + listEmpty(io_threads_list[server.io_threads_num]); + /* Wait for all threads to end their work. */ while(1) { unsigned long pending = 0; From cc90f79bafa4f9459fcb0d1e9c4a72136f2db00a Mon Sep 17 00:00:00 2001 From: WuYunlong Date: Mon, 6 Jan 2020 19:56:50 +0800 Subject: [PATCH 003/225] Fix potential memory leak of rioWriteBulkStreamID(). --- src/aof.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/aof.c b/src/aof.c index 63b34b43f..efdd68efa 100644 --- a/src/aof.c +++ b/src/aof.c @@ -1149,7 +1149,10 @@ int rioWriteBulkStreamID(rio *r,streamID *id) { int retval; sds replyid = sdscatfmt(sdsempty(),"%U-%U",id->ms,id->seq); - if ((retval = rioWriteBulkString(r,replyid,sdslen(replyid))) == 0) return 0; + if ((retval = rioWriteBulkString(r,replyid,sdslen(replyid))) == 0) { + sdsfree(replyid); + return 0; + } sdsfree(replyid); return retval; } From f2df5773b1d32477d6b4d774a2a02bdba6c0acdd Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 13 Jan 2020 12:50:26 +0100 Subject: [PATCH 004/225] A few comments about main thread serving I/O as well. Related to #6110. --- src/networking.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/networking.c b/src/networking.c index f1a6b9910..96ab8e592 100644 --- a/src/networking.c +++ b/src/networking.c @@ -2656,6 +2656,10 @@ pthread_mutex_t io_threads_mutex[IO_THREADS_MAX_NUM]; _Atomic unsigned long io_threads_pending[IO_THREADS_MAX_NUM]; int io_threads_active; /* Are the threads currently spinning waiting I/O? */ int io_threads_op; /* IO_THREADS_OP_WRITE or IO_THREADS_OP_READ. */ + +/* This is the list of clients each thread will serve when threaded I/O is + * used. We spawn N threads, and the N+1 list is used for the clients that + * are processed by the main thread itself (this is why ther is "+1"). */ list *io_threads_list[IO_THREADS_MAX_NUM+1]; void *IOThreadMain(void *myid) { @@ -2729,7 +2733,7 @@ void initThreadedIO(void) { } io_threads[i] = tid; } - io_threads_list[server.io_threads_num] = listCreate(); + io_threads_list[server.io_threads_num] = listCreate(); /* For main thread */ } void startThreadedIO(void) { @@ -2814,6 +2818,7 @@ int handleClientsWithPendingWritesUsingThreads(void) { io_threads_pending[j] = count; } + /* Also use the main thread to process a slide of clients. */ listRewind(io_threads_list[server.io_threads_num],&li); while((ln = listNext(&li))) { client *c = listNodeValue(ln); @@ -2898,6 +2903,7 @@ int handleClientsWithPendingReadsUsingThreads(void) { io_threads_pending[j] = count; } + /* Also use the main thread to process a slide of clients. */ listRewind(io_threads_list[server.io_threads_num],&li); while((ln = listNext(&li))) { client *c = listNodeValue(ln); From 47988c9666d030a84fcbb3158ce3be7bfd6e1493 Mon Sep 17 00:00:00 2001 From: WuYunlong Date: Tue, 7 Jan 2020 10:28:36 +0800 Subject: [PATCH 005/225] Fix potential memory leak of clusterLoadConfig(). --- src/cluster.c | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/src/cluster.c b/src/cluster.c index f603361cd..f9d8ae151 100644 --- a/src/cluster.c +++ b/src/cluster.c @@ -157,7 +157,10 @@ int clusterLoadConfig(char *filename) { } /* Regular config lines have at least eight fields */ - if (argc < 8) goto fmterr; + if (argc < 8) { + sdsfreesplitres(argv,argc); + goto fmterr; + } /* Create this node if it does not exist */ n = clusterLookupNode(argv[0]); @@ -166,7 +169,10 @@ int clusterLoadConfig(char *filename) { clusterAddNode(n); } /* Address and port */ - if ((p = strrchr(argv[1],':')) == NULL) goto fmterr; + if ((p = strrchr(argv[1],':')) == NULL) { + sdsfreesplitres(argv,argc); + goto fmterr; + } *p = '\0'; memcpy(n->ip,argv[1],strlen(argv[1])+1); char *port = p+1; @@ -247,7 +253,10 @@ int clusterLoadConfig(char *filename) { *p = '\0'; direction = p[1]; /* Either '>' or '<' */ slot = atoi(argv[j]+1); - if (slot < 0 || slot >= CLUSTER_SLOTS) goto fmterr; + if (slot < 0 || slot >= CLUSTER_SLOTS) { + sdsfreesplitres(argv,argc); + goto fmterr; + } p += 3; cn = clusterLookupNode(p); if (!cn) { @@ -267,8 +276,14 @@ int clusterLoadConfig(char *filename) { } else { start = stop = atoi(argv[j]); } - if (start < 0 || start >= CLUSTER_SLOTS) goto fmterr; - if (stop < 0 || stop >= CLUSTER_SLOTS) goto fmterr; + if (start < 0 || start >= CLUSTER_SLOTS) { + sdsfreesplitres(argv,argc); + goto fmterr; + } + if (stop < 0 || stop >= CLUSTER_SLOTS) { + sdsfreesplitres(argv,argc); + goto fmterr; + } while(start <= stop) clusterAddSlot(n, start++); } From ba146d4c09aeaa96cbc68874555fac983f8348f8 Mon Sep 17 00:00:00 2001 From: Vasyl Melnychuk Date: Fri, 10 Jan 2020 23:34:15 +0200 Subject: [PATCH 006/225] Make error when submitting command in incorrect context more explicit So error message `ERR only (P)SUBSCRIBE / (P)UNSUBSCRIBE / PING / QUIT allowed in this context` will become `ERR 'get' command submitted, but only (P)SUBSCRIBE / (P)UNSUBSCRIBE / PING / QUIT allowed in this context` --- src/server.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/server.c b/src/server.c index 5845a5485..3f64aaec9 100644 --- a/src/server.c +++ b/src/server.c @@ -3498,7 +3498,10 @@ int processCommand(client *c) { c->cmd->proc != unsubscribeCommand && c->cmd->proc != psubscribeCommand && c->cmd->proc != punsubscribeCommand) { - addReplyError(c,"only (P)SUBSCRIBE / (P)UNSUBSCRIBE / PING / QUIT allowed in this context"); + addReplyErrorFormat(c, + "'%s' command submitted, but only (P)SUBSCRIBE / " + "(P)UNSUBSCRIBE / PING / QUIT allowed in this context", + c->cmd->name); return C_OK; } From 1927932b437b7294ce56370b9466f9c3f050c108 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 13 Jan 2020 12:54:39 +0100 Subject: [PATCH 007/225] Port PR #6110 to new connection object code. --- src/networking.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/networking.c b/src/networking.c index 96ab8e592..73dc4afca 100644 --- a/src/networking.c +++ b/src/networking.c @@ -2822,7 +2822,7 @@ int handleClientsWithPendingWritesUsingThreads(void) { listRewind(io_threads_list[server.io_threads_num],&li); while((ln = listNext(&li))) { client *c = listNodeValue(ln); - writeToClient(c->fd,c,0); + writeToClient(c,0); } listEmpty(io_threads_list[server.io_threads_num]); @@ -2907,7 +2907,7 @@ int handleClientsWithPendingReadsUsingThreads(void) { listRewind(io_threads_list[server.io_threads_num],&li); while((ln = listNext(&li))) { client *c = listNodeValue(ln); - readQueryFromClient(NULL,c->fd,c,0); + readQueryFromClient(c->conn); } listEmpty(io_threads_list[server.io_threads_num]); From 658749cc542cb92b635486db7993da0c68129ebb Mon Sep 17 00:00:00 2001 From: WuYunlong Date: Tue, 7 Jan 2020 11:17:52 +0800 Subject: [PATCH 008/225] Free allocated sds in pfdebugCommand() to avoid memory leak. --- src/hyperloglog.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/hyperloglog.c b/src/hyperloglog.c index a44d15646..facd99743 100644 --- a/src/hyperloglog.c +++ b/src/hyperloglog.c @@ -1535,6 +1535,7 @@ void pfdebugCommand(client *c) { sds decoded = sdsempty(); if (hdr->encoding != HLL_SPARSE) { + sdsfree(decoded); addReplyError(c,"HLL encoding is not sparse"); return; } From 5be3a15a8299f028e681e14f33ff3bd82357da71 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 13 Jan 2020 18:53:12 +0100 Subject: [PATCH 009/225] Setting N I/O threads should mean N-1 additional + 1 main thread. --- src/networking.c | 47 +++++++++++++++++++++++++---------------------- 1 file changed, 25 insertions(+), 22 deletions(-) diff --git a/src/networking.c b/src/networking.c index 73dc4afca..a2e454d4b 100644 --- a/src/networking.c +++ b/src/networking.c @@ -2658,9 +2658,9 @@ int io_threads_active; /* Are the threads currently spinning waiting I/O? */ int io_threads_op; /* IO_THREADS_OP_WRITE or IO_THREADS_OP_READ. */ /* This is the list of clients each thread will serve when threaded I/O is - * used. We spawn N threads, and the N+1 list is used for the clients that - * are processed by the main thread itself (this is why ther is "+1"). */ -list *io_threads_list[IO_THREADS_MAX_NUM+1]; + * used. We spawn io_threads_num-1 threads, since one is the main thread + * itself. */ +list *io_threads_list[IO_THREADS_MAX_NUM]; void *IOThreadMain(void *myid) { /* The ID is the thread number (from 0 to server.iothreads_num-1), and is @@ -2720,12 +2720,16 @@ void initThreadedIO(void) { exit(1); } - /* Spawn the I/O threads. */ + /* Spawn and initialize the I/O threads. */ for (int i = 0; i < server.io_threads_num; i++) { + /* Things we do for all the threads including the main thread. */ + io_threads_list[i] = listCreate(); + if (i == 0) continue; /* Thread 0 is the main thread. */ + + /* Things we do only for the additional threads. */ pthread_t tid; pthread_mutex_init(&io_threads_mutex[i],NULL); io_threads_pending[i] = 0; - io_threads_list[i] = listCreate(); pthread_mutex_lock(&io_threads_mutex[i]); /* Thread will be stopped. */ if (pthread_create(&tid,NULL,IOThreadMain,(void*)(long)i) != 0) { serverLog(LL_WARNING,"Fatal: Can't initialize IO thread."); @@ -2733,14 +2737,13 @@ void initThreadedIO(void) { } io_threads[i] = tid; } - io_threads_list[server.io_threads_num] = listCreate(); /* For main thread */ } void startThreadedIO(void) { if (tio_debug) { printf("S"); fflush(stdout); } if (tio_debug) printf("--- STARTING THREADED IO ---\n"); serverAssert(io_threads_active == 0); - for (int j = 0; j < server.io_threads_num; j++) + for (int j = 1; j < server.io_threads_num; j++) pthread_mutex_unlock(&io_threads_mutex[j]); io_threads_active = 1; } @@ -2754,7 +2757,7 @@ void stopThreadedIO(void) { (int) listLength(server.clients_pending_read), (int) listLength(server.clients_pending_write)); serverAssert(io_threads_active == 1); - for (int j = 0; j < server.io_threads_num; j++) + for (int j = 1; j < server.io_threads_num; j++) pthread_mutex_lock(&io_threads_mutex[j]); io_threads_active = 0; } @@ -2805,7 +2808,7 @@ int handleClientsWithPendingWritesUsingThreads(void) { while((ln = listNext(&li))) { client *c = listNodeValue(ln); c->flags &= ~CLIENT_PENDING_WRITE; - int target_id = item_id % (server.io_threads_num+1); + int target_id = item_id % server.io_threads_num; listAddNodeTail(io_threads_list[target_id],c); item_id++; } @@ -2813,23 +2816,23 @@ int handleClientsWithPendingWritesUsingThreads(void) { /* Give the start condition to the waiting threads, by setting the * start condition atomic var. */ io_threads_op = IO_THREADS_OP_WRITE; - for (int j = 0; j < server.io_threads_num; j++) { + for (int j = 1; j < server.io_threads_num; j++) { int count = listLength(io_threads_list[j]); io_threads_pending[j] = count; } - /* Also use the main thread to process a slide of clients. */ - listRewind(io_threads_list[server.io_threads_num],&li); + /* Also use the main thread to process a slice of clients. */ + listRewind(io_threads_list[0],&li); while((ln = listNext(&li))) { client *c = listNodeValue(ln); writeToClient(c,0); } - listEmpty(io_threads_list[server.io_threads_num]); + listEmpty(io_threads_list[0]); - /* Wait for all threads to end their work. */ + /* Wait for all the other threads to end their work. */ while(1) { unsigned long pending = 0; - for (int j = 0; j < server.io_threads_num; j++) + for (int j = 1; j < server.io_threads_num; j++) pending += io_threads_pending[j]; if (pending == 0) break; } @@ -2890,7 +2893,7 @@ int handleClientsWithPendingReadsUsingThreads(void) { int item_id = 0; while((ln = listNext(&li))) { client *c = listNodeValue(ln); - int target_id = item_id % (server.io_threads_num+1); + int target_id = item_id % server.io_threads_num; listAddNodeTail(io_threads_list[target_id],c); item_id++; } @@ -2898,23 +2901,23 @@ int handleClientsWithPendingReadsUsingThreads(void) { /* Give the start condition to the waiting threads, by setting the * start condition atomic var. */ io_threads_op = IO_THREADS_OP_READ; - for (int j = 0; j < server.io_threads_num; j++) { + for (int j = 1; j < server.io_threads_num; j++) { int count = listLength(io_threads_list[j]); io_threads_pending[j] = count; } - /* Also use the main thread to process a slide of clients. */ - listRewind(io_threads_list[server.io_threads_num],&li); + /* Also use the main thread to process a slice of clients. */ + listRewind(io_threads_list[0],&li); while((ln = listNext(&li))) { client *c = listNodeValue(ln); readQueryFromClient(c->conn); } - listEmpty(io_threads_list[server.io_threads_num]); + listEmpty(io_threads_list[0]); - /* Wait for all threads to end their work. */ + /* Wait for all the other threads to end their work. */ while(1) { unsigned long pending = 0; - for (int j = 0; j < server.io_threads_num; j++) + for (int j = 1; j < server.io_threads_num; j++) pending += io_threads_pending[j]; if (pending == 0) break; } From 8e9d19bc6577cf5a8d5b05d5a5b5f1e1d14baccf Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 15 Jan 2020 17:55:24 +0100 Subject: [PATCH 010/225] Change error message for #6775. --- src/server.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/server.c b/src/server.c index 3f64aaec9..2b226f568 100644 --- a/src/server.c +++ b/src/server.c @@ -3499,8 +3499,8 @@ int processCommand(client *c) { c->cmd->proc != psubscribeCommand && c->cmd->proc != punsubscribeCommand) { addReplyErrorFormat(c, - "'%s' command submitted, but only (P)SUBSCRIBE / " - "(P)UNSUBSCRIBE / PING / QUIT allowed in this context", + "Can't execute '%s': only (P)SUBSCRIBE / " + "(P)UNSUBSCRIBE / PING / QUIT are allowed in this context", c->cmd->name); return C_OK; } From ecd17e819cbb05110a49d81c6fda10e6770a7838 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 13 Jan 2020 13:16:13 +0100 Subject: [PATCH 011/225] Jump to right label on AOF parsing error. Related to #6054. --- src/aof.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/aof.c b/src/aof.c index 9c2fa838b..63b34b43f 100644 --- a/src/aof.c +++ b/src/aof.c @@ -789,12 +789,14 @@ int loadAppendOnlyFile(char *filename) { for (j = 0; j < argc; j++) { /* Parse the argument len. */ - if (fgets(buf,sizeof(buf),fp) == NULL || - buf[0] != '$') - { + char *readres = fgets(buf,sizeof(buf),fp); + if (readres == NULL || buf[0] != '$') { fakeClient->argc = j; /* Free up to j-1. */ freeFakeClientArgv(fakeClient); - goto readerr; + if (readres == NULL) + goto readerr; + else + goto fmterr; } len = strtol(buf+1,NULL,10); From cbabf779c24fd107ca8e4fe6369e4677ad0eda10 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 13 Jan 2020 13:25:37 +0100 Subject: [PATCH 012/225] Simplify #6379 changes. --- src/aof.c | 5 +---- src/cluster.c | 8 +++----- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/src/aof.c b/src/aof.c index efdd68efa..9eeb3f1e2 100644 --- a/src/aof.c +++ b/src/aof.c @@ -1149,10 +1149,7 @@ int rioWriteBulkStreamID(rio *r,streamID *id) { int retval; sds replyid = sdscatfmt(sdsempty(),"%U-%U",id->ms,id->seq); - if ((retval = rioWriteBulkString(r,replyid,sdslen(replyid))) == 0) { - sdsfree(replyid); - return 0; - } + retval = rioWriteBulkString(r,replyid,sdslen(replyid)); sdsfree(replyid); return retval; } diff --git a/src/cluster.c b/src/cluster.c index f9d8ae151..c05e46f76 100644 --- a/src/cluster.c +++ b/src/cluster.c @@ -276,11 +276,9 @@ int clusterLoadConfig(char *filename) { } else { start = stop = atoi(argv[j]); } - if (start < 0 || start >= CLUSTER_SLOTS) { - sdsfreesplitres(argv,argc); - goto fmterr; - } - if (stop < 0 || stop >= CLUSTER_SLOTS) { + if (start < 0 || start >= CLUSTER_SLOTS || + stop < 0 || stop >= CLUSTER_SLOTS) + { sdsfreesplitres(argv,argc); goto fmterr; } From 721a39ddff0eb9be822d27dba80267f6da149aff Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 13 Jan 2020 19:10:42 +0100 Subject: [PATCH 013/225] Document I/O threads in redis.conf. --- redis.conf | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/redis.conf b/redis.conf index 5f9547c1d..07005cffe 100644 --- a/redis.conf +++ b/redis.conf @@ -938,6 +938,52 @@ lazyfree-lazy-expire no lazyfree-lazy-server-del no replica-lazy-flush no +################################ THREADED I/O ################################# + +# Redis is mostly single threaded, however there are certain threaded +# operations such as UNLINK, slow I/O accesses and other things that are +# performed on side threads. +# +# Now it is also possible to handle Redis clients socket reads and writes +# in different I/O threads. Since especially writing is so slow, normally +# Redis users use pipelining in order to speedup the Redis performances per +# core, and spawn multiple instances in order to scale more. Using I/O +# threads it is possible to easily speedup two times Redis without resorting +# to pipelining nor sharding of the instance. +# +# By default threading is disabled, we suggest enabling it only in machines +# that have at least 4 or more cores, leaving at least one spare core. +# Using more than 8 threads is unlikely to help much. We also recommend using +# threaded I/O only if you actually have performance problems, with Redis +# instances being able to use a quite big percentage of CPU time, otherwise +# there is no point in using this feature. +# +# So for instance if you have a four cores boxes, try to use 2 or 3 I/O +# threads, if you have a 8 cores, try to use 6 threads. In order to +# enable I/O threads use the following configuration directive: +# +# io-threads 4 +# +# Setting io-threads to 1 will just use the main thread as usually. +# When I/O threads are enabled, we only use threads for writes, that is +# to thread the write(2) syscall and transfer the client buffers to the +# socket. However it is also possible to enable threading of reads and +# protocol parsing using the following configuration directive, by setting +# it to yes: +# +# io-threads-do-reads no +# +# Usually threading reads doesn't help much. +# +# NOTE 1: This configuration directive cannot be changed at runtime via +# CONFIG SET. Aso this feature currently does not work when SSL is +# enabled. +# +# NOTE 2: If you want to test the Redis speedup using redis-benchmark, make +# sure you also run the benchmark itself in threaded mode, using the +# --threads option to match the number of Redis theads, otherwise you'll not +# be able to notice the improvements. + ############################## APPEND ONLY MODE ############################### # By default Redis asynchronously dumps the dataset on disk. This mode is From ec0c61da05db61fccd24905387f2ca95828902a1 Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Mon, 3 Feb 2020 15:58:28 +0200 Subject: [PATCH 014/225] fix uninitialized info_cb var in module.c --- src/module.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/module.c b/src/module.c index d922c5c20..914c50df3 100644 --- a/src/module.c +++ b/src/module.c @@ -859,6 +859,7 @@ void RM_SetModuleAttribs(RedisModuleCtx *ctx, const char *name, int ver, int api module->in_call = 0; module->in_hook = 0; module->options = 0; + module->info_cb = 0; ctx->module = module; } From 29d4a1502a80afa92079897a57836f5a7e57586b Mon Sep 17 00:00:00 2001 From: Yossi Gottlieb Date: Wed, 29 Jan 2020 21:40:02 +0200 Subject: [PATCH 015/225] TLS: Fix missing initialization in redis-cli. --- src/redis-cli.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/redis-cli.c b/src/redis-cli.c index 065c389c6..1919829e1 100644 --- a/src/redis-cli.c +++ b/src/redis-cli.c @@ -49,6 +49,7 @@ #include #ifdef USE_OPENSSL #include +#include #include #endif #include /* use sds.h from hiredis, so that only one set of sds functions will be present in the binary */ @@ -7933,6 +7934,14 @@ int main(int argc, char **argv) { parseEnv(); +#ifdef USE_OPENSSL + if (config.tls) { + ERR_load_crypto_strings(); + SSL_load_error_strings(); + SSL_library_init(); + } +#endif + /* Cluster Manager mode */ if (CLUSTER_MANAGER_MODE()) { clusterManagerCommandProc *proc = validateClusterManagerCommand(); From d2509811b7074a14113e506a434308abb7f54246 Mon Sep 17 00:00:00 2001 From: WuYunlong Date: Tue, 4 Feb 2020 16:34:11 +0800 Subject: [PATCH 016/225] Add tcl regression test in scripting.tcl to reproduce memory leak. --- tests/unit/scripting.tcl | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/unit/scripting.tcl b/tests/unit/scripting.tcl index 2543a0377..fb36d0b80 100644 --- a/tests/unit/scripting.tcl +++ b/tests/unit/scripting.tcl @@ -741,3 +741,8 @@ start_server {tags {"scripting repl"}} { } } +start_server {tags {"scripting"}} { + r script debug sync + r eval {return 'hello'} 0 + r eval {return 'hello'} 0 +} From 40295fb3fe9953283796dac03196b4e78e6732ef Mon Sep 17 00:00:00 2001 From: Guy Benoish Date: Tue, 24 Dec 2019 17:14:23 +0530 Subject: [PATCH 017/225] Modules: Fix blocked-client-related memory leak If a blocked module client times-out (or disconnects, unblocked by CLIENT command, etc.) we need to call moduleUnblockClient in order to free memory allocated by the module sub-system and blocked-client private data Other changes: Made blockedonkeys.tcl tests a bit more aggressive in order to smoke-out potential memory leaks --- src/module.c | 9 +++++++ tests/modules/blockonkeys.c | 13 ++++++----- tests/unit/moduleapi/blockonkeys.tcl | 35 ++++++++++++++++++++++++++++ 3 files changed, 51 insertions(+), 6 deletions(-) diff --git a/src/module.c b/src/module.c index 965bb4460..d2b267be2 100644 --- a/src/module.c +++ b/src/module.c @@ -4277,6 +4277,15 @@ void unblockClientFromModule(client *c) { moduleFreeContext(&ctx); } + /* If we made it here and client is still blocked it means that the command + * timed-out, client was killed or disconnected and disconnect_callback was + * not implemented (or it was, but RM_UnblockClient was not called from + * within it, as it should). + * We must call moduleUnblockClient in order to free privdata and + * RedisModuleBlockedClient */ + if (!bc->unblocked) + moduleUnblockClient(c); + bc->client = NULL; /* Reset the client for a new query since, for blocking commands implemented * into modules, we do not it immediately after the command returns (and diff --git a/tests/modules/blockonkeys.c b/tests/modules/blockonkeys.c index 959918b1c..10dc65b1a 100644 --- a/tests/modules/blockonkeys.c +++ b/tests/modules/blockonkeys.c @@ -172,13 +172,13 @@ int bpopgt_reply_callback(RedisModuleCtx *ctx, RedisModuleString **argv, int arg REDISMODULE_NOT_USED(argv); REDISMODULE_NOT_USED(argc); RedisModuleString *keyname = RedisModule_GetBlockedClientReadyKey(ctx); - long long gt = (long long)RedisModule_GetBlockedClientPrivateData(ctx); + long long *pgt = RedisModule_GetBlockedClientPrivateData(ctx); fsl_t *fsl; if (!get_fsl(ctx, keyname, REDISMODULE_READ, 0, &fsl, 0)) return REDISMODULE_ERR; - if (!fsl || fsl->list[fsl->length-1] <= gt) + if (!fsl || fsl->list[fsl->length-1] <= *pgt) return REDISMODULE_ERR; RedisModule_ReplyWithLongLong(ctx, fsl->list[--fsl->length]); @@ -192,10 +192,8 @@ int bpopgt_timeout_callback(RedisModuleCtx *ctx, RedisModuleString **argv, int a } void bpopgt_free_privdata(RedisModuleCtx *ctx, void *privdata) { - /* Nothing to do because privdata is actually a 'long long', - * not a pointer to the heap */ REDISMODULE_NOT_USED(ctx); - REDISMODULE_NOT_USED(privdata); + RedisModule_Free(privdata); } /* FSL.BPOPGT - Block clients until list has an element greater than . @@ -217,9 +215,12 @@ int fsl_bpopgt(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { return REDISMODULE_OK; if (!fsl || fsl->list[fsl->length-1] <= gt) { + /* We use malloc so the tests in blockedonkeys.tcl can check for memory leaks */ + long long *pgt = RedisModule_Alloc(sizeof(long long)); + *pgt = gt; /* Key is empty or has <2 elements, we must block */ RedisModule_BlockClientOnKeys(ctx, bpopgt_reply_callback, bpopgt_timeout_callback, - bpopgt_free_privdata, timeout, &argv[1], 1, (void*)gt); + bpopgt_free_privdata, timeout, &argv[1], 1, pgt); } else { RedisModule_ReplyWithLongLong(ctx, fsl->list[--fsl->length]); } diff --git a/tests/unit/moduleapi/blockonkeys.tcl b/tests/unit/moduleapi/blockonkeys.tcl index cb99ab1c9..b380227e0 100644 --- a/tests/unit/moduleapi/blockonkeys.tcl +++ b/tests/unit/moduleapi/blockonkeys.tcl @@ -45,18 +45,24 @@ start_server {tags {"modules"}} { test {Module client blocked on keys (with metadata): Timeout} { r del k set rd [redis_deferring_client] + $rd client id + set cid [$rd read] r fsl.push k 33 $rd fsl.bpopgt k 35 1 assert_equal {Request timedout} [$rd read] + r client kill id $cid ;# try to smoke-out client-related memory leak } test {Module client blocked on keys (with metadata): Blocked, case 1} { r del k set rd [redis_deferring_client] + $rd client id + set cid [$rd read] r fsl.push k 33 $rd fsl.bpopgt k 33 0 r fsl.push k 34 assert_equal {34} [$rd read] + r client kill id $cid ;# try to smoke-out client-related memory leak } test {Module client blocked on keys (with metadata): Blocked, case 2} { @@ -70,6 +76,35 @@ start_server {tags {"modules"}} { assert_equal {36} [$rd read] } + test {Module client blocked on keys (with metadata): Blocked, CLIENT KILL} { + r del k + set rd [redis_deferring_client] + $rd client id + set cid [$rd read] + $rd fsl.bpopgt k 35 0 + r client kill id $cid ;# try to smoke-out client-related memory leak + } + + test {Module client blocked on keys (with metadata): Blocked, CLIENT UNBLOCK TIMEOUT} { + r del k + set rd [redis_deferring_client] + $rd client id + set cid [$rd read] + $rd fsl.bpopgt k 35 0 + r client unblock $cid timeout ;# try to smoke-out client-related memory leak + assert_equal {Request timedout} [$rd read] + } + + test {Module client blocked on keys (with metadata): Blocked, CLIENT UNBLOCK ERROR} { + r del k + set rd [redis_deferring_client] + $rd client id + set cid [$rd read] + $rd fsl.bpopgt k 35 0 + r client unblock $cid error ;# try to smoke-out client-related memory leak + assert_error "*unblocked*" {$rd read} + } + test {Module client blocked on keys does not wake up on wrong type} { r del k set rd [redis_deferring_client] From eecfa9793e27d9aec2808e208422859dbcc49af4 Mon Sep 17 00:00:00 2001 From: WuYunlong Date: Tue, 4 Feb 2020 16:38:46 +0800 Subject: [PATCH 018/225] Fix lua related memory leak. --- src/scripting.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/scripting.c b/src/scripting.c index 9282b7fd9..a5c59b113 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -2473,6 +2473,7 @@ void ldbEval(lua_State *lua, sds *argv, int argc) { ldbLog(sdscatfmt(sdsempty()," %s",lua_tostring(lua,-1))); lua_pop(lua,1); sdsfree(code); + sdsfree(expr); return; } } From bbce3ba974d2f29240c569a7b30aaec14eba3e2b Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 29 Jan 2020 12:47:50 +0100 Subject: [PATCH 019/225] Add more info in the unblockClientFromModule() function. --- src/module.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/module.c b/src/module.c index d2b267be2..7fdab1b34 100644 --- a/src/module.c +++ b/src/module.c @@ -4282,7 +4282,13 @@ void unblockClientFromModule(client *c) { * not implemented (or it was, but RM_UnblockClient was not called from * within it, as it should). * We must call moduleUnblockClient in order to free privdata and - * RedisModuleBlockedClient */ + * RedisModuleBlockedClient. + * + * Note that clients implementing threads and working with private data, + * should make sure to stop the threads or protect the private data + * in some other way in the disconnection and timeout callback, because + * here we are going to free the private data associated with the + * blocked client. */ if (!bc->unblocked) moduleUnblockClient(c); From f7a94526dd93ebb56b1a21ba86a71329c4e3f6e4 Mon Sep 17 00:00:00 2001 From: Leo Murillo Date: Sun, 2 Feb 2020 02:48:00 -0600 Subject: [PATCH 020/225] Set ZSKIPLIST_MAXLEVEL to optimal value given 2^64 elements and p=0.25 --- src/server.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/server.h b/src/server.h index 8e354c03d..5d63e9b55 100644 --- a/src/server.h +++ b/src/server.h @@ -335,7 +335,7 @@ typedef long long ustime_t; /* microsecond time type. */ /* Anti-warning macro... */ #define UNUSED(V) ((void) V) -#define ZSKIPLIST_MAXLEVEL 64 /* Should be enough for 2^64 elements */ +#define ZSKIPLIST_MAXLEVEL 32 /* Should be enough for 2^64 elements */ #define ZSKIPLIST_P 0.25 /* Skiplist P = 1/4 */ /* Append only defines */ From 6fe55c2f299bbb306533af39cb28346bf5d473d3 Mon Sep 17 00:00:00 2001 From: Guy Benoish Date: Thu, 30 Jan 2020 18:14:45 +0530 Subject: [PATCH 021/225] ld2string should fail if string contains \0 in the middle This bug affected RM_StringToLongDouble and HINCRBYFLOAT. I added tests for both cases. Main changes: 1. Fixed string2ld to fail if string contains \0 in the middle 2. Use string2ld in getLongDoubleFromObject - No point of having duplicated code here The two changes above broke RM_SaveLongDouble/RM_LoadLongDouble because the long double string was saved with length+1 (An innocent mistake, but it's actually a bug - The length passed to RM_SaveLongDouble should not include the last \0). --- src/module.c | 2 +- src/object.c | 10 +--------- src/util.c | 3 ++- tests/modules/misc.c | 9 +++++++++ tests/unit/type/hash.tcl | 7 +++++++ 5 files changed, 20 insertions(+), 11 deletions(-) diff --git a/src/module.c b/src/module.c index 7fdab1b34..d922c5c20 100644 --- a/src/module.c +++ b/src/module.c @@ -3901,7 +3901,7 @@ void RM_SaveLongDouble(RedisModuleIO *io, long double value) { /* Long double has different number of bits in different platforms, so we * save it as a string type. */ size_t len = ld2string(buf,sizeof(buf),value,LD_STR_HEX); - RM_SaveStringBuffer(io,buf,len+1); /* len+1 for '\0' */ + RM_SaveStringBuffer(io,buf,len); } /* In the context of the rdb_save method of a module data type, loads back the diff --git a/src/object.c b/src/object.c index 2201a317a..52007b474 100644 --- a/src/object.c +++ b/src/object.c @@ -640,21 +640,13 @@ int getDoubleFromObjectOrReply(client *c, robj *o, double *target, const char *m int getLongDoubleFromObject(robj *o, long double *target) { long double value; - char *eptr; if (o == NULL) { value = 0; } else { serverAssertWithInfo(NULL,o,o->type == OBJ_STRING); if (sdsEncodedObject(o)) { - errno = 0; - value = strtold(o->ptr, &eptr); - if (sdslen(o->ptr) == 0 || - isspace(((const char*)o->ptr)[0]) || - (size_t)(eptr-(char*)o->ptr) != sdslen(o->ptr) || - (errno == ERANGE && - (value == HUGE_VAL || value == -HUGE_VAL || value == 0)) || - isnan(value)) + if (!string2ld(o->ptr, sdslen(o->ptr), &value)) return C_ERR; } else if (o->encoding == OBJ_ENCODING_INT) { value = (long)o->ptr; diff --git a/src/util.c b/src/util.c index 20471b539..2be42a0df 100644 --- a/src/util.c +++ b/src/util.c @@ -471,13 +471,14 @@ int string2ld(const char *s, size_t slen, long double *dp) { long double value; char *eptr; - if (slen >= sizeof(buf)) return 0; + if (slen == 0 || slen >= sizeof(buf)) return 0; memcpy(buf,s,slen); buf[slen] = '\0'; errno = 0; value = strtold(buf, &eptr); if (isspace(buf[0]) || eptr[0] != '\0' || + (size_t)(eptr-buf) != slen || (errno == ERANGE && (value == HUGE_VAL || value == -HUGE_VAL || value == 0)) || errno == EINVAL || diff --git a/tests/modules/misc.c b/tests/modules/misc.c index b5a032f60..41bec06ed 100644 --- a/tests/modules/misc.c +++ b/tests/modules/misc.c @@ -74,6 +74,15 @@ int test_ld_conv(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { RedisModule_ReplyWithError(ctx, err); goto final; } + /* Make sure we can't convert a string that has \0 in it */ + char buf[4] = "123"; + buf[1] = '\0'; + RedisModuleString *s3 = RedisModule_CreateString(ctx, buf, 3); + long double ld3; + if (RedisModule_StringToLongDouble(s3, &ld3) == REDISMODULE_OK) { + RedisModule_ReplyWithError(ctx, "Invalid string successfully converted to long double"); + goto final; + } RedisModule_ReplyWithLongDouble(ctx, ld2); final: RedisModule_FreeString(ctx, s1); diff --git a/tests/unit/type/hash.tcl b/tests/unit/type/hash.tcl index d2c679d32..9f8a21b1c 100644 --- a/tests/unit/type/hash.tcl +++ b/tests/unit/type/hash.tcl @@ -390,6 +390,13 @@ start_server {tags {"hash"}} { lappend rv [string match "ERR*not*float*" $bigerr] } {1 1} + test {HINCRBYFLOAT fails against hash value that contains a null-terminator in the middle} { + r hset h f "1\x002" + catch {r hincrbyfloat h f 1} err + set rv {} + lappend rv [string match "ERR*not*float*" $err] + } {1} + test {HSTRLEN against the small hash} { set err {} foreach k [array names smallhash *] { From 577fc4388b4701d78244ec37d4dd8a5e7abe35ca Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 27 Jan 2020 18:37:52 +0100 Subject: [PATCH 022/225] ACL LOG: data structures and initial functions. --- src/acl.c | 51 ++++++++++++++++++++++++++++++++++++++++++++++++- src/multi.c | 2 +- src/scripting.c | 2 +- src/server.c | 2 +- src/server.h | 2 +- 5 files changed, 54 insertions(+), 5 deletions(-) diff --git a/src/acl.c b/src/acl.c index 1f395bd3f..4391382a6 100644 --- a/src/acl.c +++ b/src/acl.c @@ -49,6 +49,8 @@ list *UsersToLoad; /* This is a list of users found in the configuration file array of SDS pointers: the first is the user name, all the remaining pointers are ACL rules in the same format as ACLSetUser(). */ +list *ACLLog; /* Our security log, the user is able to inspect that + using the ACL LOG command .*/ struct ACLCategoryItem { const char *name; @@ -920,6 +922,7 @@ void ACLInitDefaultUser(void) { void ACLInit(void) { Users = raxNew(); UsersToLoad = listCreate(); + ACLLog = listCreate(); ACLInitDefaultUser(); } @@ -1034,7 +1037,7 @@ user *ACLGetUserByName(const char *name, size_t namelen) { * command cannot be executed because the user is not allowed to run such * command, the second if the command is denied because the user is trying * to access keys that are not among the specified patterns. */ -int ACLCheckCommandPerm(client *c) { +int ACLCheckCommandPerm(client *c, int *keyidxptr) { user *u = c->user; uint64_t id = c->cmd->id; @@ -1094,6 +1097,7 @@ int ACLCheckCommandPerm(client *c) { } } if (!match) { + if (keyidxptr) *keyidxptr = keyidx[j]; getKeysFreeResult(keyidx); return ACL_DENIED_KEY; } @@ -1454,6 +1458,51 @@ void ACLLoadUsersAtStartup(void) { } } +/* ============================================================================= + * ACL log + * ==========================================================================*/ + +#define ACL_LOG_CTX_TOPLEVEL 0 +#define ACL_LOG_CTX_LUA 1 +#define ACL_LOG_CTX_MULTI 2 + +/* This structure defines an entry inside the ACL log. */ +typedef struct aclLogEntry { + uint64_t count; /* Number of times this happened recently. */ + int reason; /* Reason for denying the command. ACL_DENIED_*. */ + int context; /* Toplevel, Lua or MULTI/EXEC? ACL_LOG_CTX_*. */ + sds object; /* The key name or command name. */ + sds username; /* User the client is authenticated with. */ + mstime_t ctime; /* Milliseconds time of last update to this entry. */ + sds cinfo; /* Client info (last client if updated). */ +} aclLogEntry; + +void addACLLogEntry(client *c, int reason, int keypos) { + /* Create a new entry. */ + struct aclLogEntry *le = zmalloc(sizeof(*le)); + le->count = 1; + le->object = (reason == ACL_DENIED_CMD) ? sdsnew(c->cmd->name) : + sdsdup(c->argv[keypos]->ptr); + le->username = sdsdup(c->user->name); + le->ctime = mstime(); + + client *realclient = c; + if (realclient->flags & CLIENT_LUA) realclient = server.lua_caller; + + le->cinfo = catClientInfoString(sdsempty(),realclient); + if (c->flags & CLIENT_MULTI) { + le->context = ACL_LOG_CTX_MULTI; + } else if (c->flags & CLIENT_LUA) { + le->context = ACL_LOG_CTX_LUA; + } else { + le->context = ACL_LOG_CTX_TOPLEVEL; + } + + /* Add it to our list of entires. We'll have to trim the list + * to its maximum size. */ + listAddNodeHead(ACLLog, le); +} + /* ============================================================================= * ACL related commands * ==========================================================================*/ diff --git a/src/multi.c b/src/multi.c index df11225bd..640149870 100644 --- a/src/multi.c +++ b/src/multi.c @@ -177,7 +177,7 @@ void execCommand(client *c) { must_propagate = 1; } - int acl_retval = ACLCheckCommandPerm(c); + int acl_retval = ACLCheckCommandPerm(c,NULL); if (acl_retval != ACL_OK) { addReplyErrorFormat(c, "-NOPERM ACLs rules changed between the moment the " diff --git a/src/scripting.c b/src/scripting.c index a5c59b113..81e174870 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -606,7 +606,7 @@ int luaRedisGenericCommand(lua_State *lua, int raise_error) { } /* Check the ACLs. */ - int acl_retval = ACLCheckCommandPerm(c); + int acl_retval = ACLCheckCommandPerm(c,NULL); if (acl_retval != ACL_OK) { if (acl_retval == ACL_DENIED_CMD) luaPushError(lua, "The user executing the script can't run this " diff --git a/src/server.c b/src/server.c index 2b226f568..b5e27e238 100644 --- a/src/server.c +++ b/src/server.c @@ -3377,7 +3377,7 @@ int processCommand(client *c) { /* Check if the user can run this command according to the current * ACLs. */ - int acl_retval = ACLCheckCommandPerm(c); + int acl_retval = ACLCheckCommandPerm(c,NULL); if (acl_retval != ACL_OK) { flagTransaction(c); if (acl_retval == ACL_DENIED_CMD) diff --git a/src/server.h b/src/server.h index 5d63e9b55..fe27cf232 100644 --- a/src/server.h +++ b/src/server.h @@ -1824,7 +1824,7 @@ int ACLCheckUserCredentials(robj *username, robj *password); int ACLAuthenticateUser(client *c, robj *username, robj *password); unsigned long ACLGetCommandID(const char *cmdname); user *ACLGetUserByName(const char *name, size_t namelen); -int ACLCheckCommandPerm(client *c); +int ACLCheckCommandPerm(client *c, int *keyidxptr); int ACLSetUser(user *u, const char *op, ssize_t oplen); sds ACLDefaultUserFirstPassword(void); uint64_t ACLGetCommandCategoryFlagByName(const char *name); From d9b153c9f66877a659ad2c87d178490cbd377749 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 28 Jan 2020 17:30:50 +0100 Subject: [PATCH 023/225] ACL LOG: implement ACL LOG subcommadn skeleton. --- src/acl.c | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/src/acl.c b/src/acl.c index 4391382a6..a166469d0 100644 --- a/src/acl.c +++ b/src/acl.c @@ -1477,6 +1477,11 @@ typedef struct aclLogEntry { sds cinfo; /* Client info (last client if updated). */ } aclLogEntry; +/* Adds a new entry in the ACL log, making sure to delete the old entry + * if we reach the maximum length allowed for the log. This function attempts + * to find similar entries in the current log in order to bump the counter of + * the log entry instead of creating many entries for very similar ACL + * rules issues. */ void addACLLogEntry(client *c, int reason, int keypos) { /* Create a new entry. */ struct aclLogEntry *le = zmalloc(sizeof(*le)); @@ -1518,6 +1523,7 @@ void addACLLogEntry(client *c, int reason, int keypos) { * ACL GETUSER * ACL GENPASS * ACL WHOAMI + * ACL LOG [ | RESET] */ void aclCommand(client *c) { char *sub = c->argv[1]->ptr; @@ -1704,6 +1710,36 @@ void aclCommand(client *c) { char pass[32]; /* 128 bits of actual pseudo random data. */ getRandomHexChars(pass,sizeof(pass)); addReplyBulkCBuffer(c,pass,sizeof(pass)); + } else if (!strcasecmp(sub,"log") && (c->argc == 2 || c->argc ==3)) { + long count = 10; /* Number of entries to emit by default. */ + + /* Parse the only argument that LOG may have: it could be either + * the number of entires the user wants to display, or alternatively + * the "RESET" command in order to flush the old entires. */ + if (c->argc == 3) { + if (!strcasecmp(c->argv[2]->ptr,"reset")) { + /* TODO: reset the log. */ + addReply(c,shared.ok); + return; + } else if (getLongFromObjectOrReply(c,c->argv[2],&count,NULL) + != C_OK) + { + return; + } + if (count < 0) count = 0; + } + + /* Fix the count according to the number of entries we got. */ + if ((size_t)count > listLength(ACLLog)) + count = listLength(ACLLog); + + addReplyArrayLen(c,count); + listIter li; + listNode *ln; + listRewind(ACLLog,&li); + while (count-- && (ln = listNext(&li)) != NULL) { + addReplyLongLong(c,1234); + } } else if (!strcasecmp(sub,"help")) { const char *help[] = { "LOAD -- Reload users from the ACL file.", @@ -1716,6 +1752,7 @@ void aclCommand(client *c) { "CAT -- List commands inside category.", "GENPASS -- Generate a secure user password.", "WHOAMI -- Return the current connection username.", +"LOG [ | RESET] -- Show the ACL log entries.", NULL }; addReplyHelp(c,help); From f1974d5d67a4b5489cfc4225da7bfe13ba10b563 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 28 Jan 2020 18:04:20 +0100 Subject: [PATCH 024/225] ACL LOG: actually emit entries. --- src/acl.c | 34 ++++++++++++++++++++++++++++++---- src/server.c | 4 +++- src/server.h | 1 + 3 files changed, 34 insertions(+), 5 deletions(-) diff --git a/src/acl.c b/src/acl.c index a166469d0..5937c069c 100644 --- a/src/acl.c +++ b/src/acl.c @@ -1467,7 +1467,7 @@ void ACLLoadUsersAtStartup(void) { #define ACL_LOG_CTX_MULTI 2 /* This structure defines an entry inside the ACL log. */ -typedef struct aclLogEntry { +typedef struct ACLLogEntry { uint64_t count; /* Number of times this happened recently. */ int reason; /* Reason for denying the command. ACL_DENIED_*. */ int context; /* Toplevel, Lua or MULTI/EXEC? ACL_LOG_CTX_*. */ @@ -1475,7 +1475,7 @@ typedef struct aclLogEntry { sds username; /* User the client is authenticated with. */ mstime_t ctime; /* Milliseconds time of last update to this entry. */ sds cinfo; /* Client info (last client if updated). */ -} aclLogEntry; +} ACLLogEntry; /* Adds a new entry in the ACL log, making sure to delete the old entry * if we reach the maximum length allowed for the log. This function attempts @@ -1484,8 +1484,9 @@ typedef struct aclLogEntry { * rules issues. */ void addACLLogEntry(client *c, int reason, int keypos) { /* Create a new entry. */ - struct aclLogEntry *le = zmalloc(sizeof(*le)); + struct ACLLogEntry *le = zmalloc(sizeof(*le)); le->count = 1; + le->reason = reason; le->object = (reason == ACL_DENIED_CMD) ? sdsnew(c->cmd->name) : sdsdup(c->argv[keypos]->ptr); le->username = sdsdup(c->user->name); @@ -1737,8 +1738,33 @@ void aclCommand(client *c) { listIter li; listNode *ln; listRewind(ACLLog,&li); + mstime_t now = mstime(); while (count-- && (ln = listNext(&li)) != NULL) { - addReplyLongLong(c,1234); + ACLLogEntry *le = listNodeValue(ln); + addReplyMapLen(c,7); + addReplyBulkCString(c,"count"); + addReplyLongLong(c,le->count); + addReplyBulkCString(c,"reason"); + addReplyBulkCString(c,(le->reason == ACL_DENIED_CMD) ? + "command" : "key"); + char *ctxstr; + switch(le->context) { + case ACL_LOG_CTX_TOPLEVEL: ctxstr="toplevel"; break; + case ACL_LOG_CTX_MULTI: ctxstr="multi"; break; + case ACL_LOG_CTX_LUA: ctxstr="lua"; break; + default: ctxstr="unknown"; + } + addReplyBulkCString(c,"context"); + addReplyBulkCString(c,ctxstr); + addReplyBulkCString(c,"object"); + addReplyBulkCBuffer(c,le->object,sdslen(le->object)); + addReplyBulkCString(c,"username"); + addReplyBulkCBuffer(c,le->username,sdslen(le->username)); + addReplyBulkCString(c,"age-seconds"); + double age = (double)(now - le->ctime)/1000; + addReplyDouble(c,age); + addReplyBulkCString(c,"client-info"); + addReplyBulkCBuffer(c,le->cinfo,sdslen(le->cinfo)); } } else if (!strcasecmp(sub,"help")) { const char *help[] = { diff --git a/src/server.c b/src/server.c index b5e27e238..6968f311f 100644 --- a/src/server.c +++ b/src/server.c @@ -3377,8 +3377,10 @@ int processCommand(client *c) { /* Check if the user can run this command according to the current * ACLs. */ - int acl_retval = ACLCheckCommandPerm(c,NULL); + int acl_keypos; + int acl_retval = ACLCheckCommandPerm(c,&acl_keypos); if (acl_retval != ACL_OK) { + addACLLogEntry(c,acl_retval,acl_keypos); flagTransaction(c); if (acl_retval == ACL_DENIED_CMD) addReplyErrorFormat(c, diff --git a/src/server.h b/src/server.h index fe27cf232..228a14f34 100644 --- a/src/server.h +++ b/src/server.h @@ -1836,6 +1836,7 @@ void ACLLoadUsersAtStartup(void); void addReplyCommandCategories(client *c, struct redisCommand *cmd); user *ACLCreateUnlinkedUser(); void ACLFreeUserAndKillClients(user *u); +void addACLLogEntry(client *c, int reason, int keypos); /* Sorted sets data type */ From e271a61103b8b57371e3283eddff5ef1b7b09716 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 29 Jan 2020 18:40:32 +0100 Subject: [PATCH 025/225] ACL LOG: group similar entries in a given time delta. --- src/acl.c | 61 ++++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 58 insertions(+), 3 deletions(-) diff --git a/src/acl.c b/src/acl.c index 5937c069c..d03599a79 100644 --- a/src/acl.c +++ b/src/acl.c @@ -1465,6 +1465,7 @@ void ACLLoadUsersAtStartup(void) { #define ACL_LOG_CTX_TOPLEVEL 0 #define ACL_LOG_CTX_LUA 1 #define ACL_LOG_CTX_MULTI 2 +#define ACL_LOG_GROUPING_MAX_TIME_DELTA 60000 /* This structure defines an entry inside the ACL log. */ typedef struct ACLLogEntry { @@ -1477,6 +1478,28 @@ typedef struct ACLLogEntry { sds cinfo; /* Client info (last client if updated). */ } ACLLogEntry; +/* This function will check if ACL entries 'a' and 'b' are similar enough + * that we should actually update the existing entry in our ACL log instead + * of creating a new one. */ +int ACLLogMatchEntry(ACLLogEntry *a, ACLLogEntry *b) { + if (a->reason != b->reason) return 0; + if (a->context != b->context) return 0; + mstime_t delta = a->ctime - b->ctime; + if (delta < 0) delta = -delta; + if (delta > ACL_LOG_GROUPING_MAX_TIME_DELTA) return 0; + if (sdscmp(a->object,b->object) != 0) return 0; + if (sdscmp(a->username,b->username) != 0) return 0; + return 1; +} + +/* Release an ACL log entry. */ +void ACLFreeLogEntry(ACLLogEntry *le) { + sdsfree(le->object); + sdsfree(le->username); + sdsfree(le->cinfo); + zfree(le); +} + /* Adds a new entry in the ACL log, making sure to delete the old entry * if we reach the maximum length allowed for the log. This function attempts * to find similar entries in the current log in order to bump the counter of @@ -1504,9 +1527,41 @@ void addACLLogEntry(client *c, int reason, int keypos) { le->context = ACL_LOG_CTX_TOPLEVEL; } - /* Add it to our list of entires. We'll have to trim the list - * to its maximum size. */ - listAddNodeHead(ACLLog, le); + /* Try to match this entry with past ones, to see if we can just + * update an existing entry instead of creating a new one. */ + long toscan = 10; /* Do a limited work trying to find duplicated. */ + listIter li; + listNode *ln; + listRewind(ACLLog,&li); + ACLLogEntry *match = NULL; + while (toscan-- && (ln = listNext(&li)) != NULL) { + ACLLogEntry *current = listNodeValue(ln); + if (ACLLogMatchEntry(current,le)) { + match = current; + listDelNode(ACLLog,ln); + listAddNodeHead(ACLLog,current); + break; + } + } + + /* If there is a match update the entry, otherwise add it as a + * new one. */ + if (match) { + /* We update a few fields of the existing entry and bump the + * counter of events for this entry. */ + sdsfree(match->cinfo); + match->cinfo = le->cinfo; + match->ctime = le->ctime; + match->count++; + + /* Release the old entry. */ + le->cinfo = NULL; + ACLFreeLogEntry(le); + } else { + /* Add it to our list of entires. We'll have to trim the list + * to its maximum size. */ + listAddNodeHead(ACLLog, le); + } } /* ============================================================================= From 943008ebac4ef23d30a2a32dd2900e25794811d5 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 29 Jan 2020 18:51:04 +0100 Subject: [PATCH 026/225] ACL LOG: implement LOG RESET. --- src/acl.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/acl.c b/src/acl.c index d03599a79..16acd4d3e 100644 --- a/src/acl.c +++ b/src/acl.c @@ -95,6 +95,7 @@ struct ACLUserFlag { void ACLResetSubcommandsForCommand(user *u, unsigned long id); void ACLResetSubcommands(user *u); void ACLAddAllowedSubcommand(user *u, unsigned long id, const char *sub); +void ACLFreeLogEntry(void *le); /* The length of the string representation of a hashed password. */ #define HASH_PASSWORD_LEN SHA256_BLOCK_SIZE*2 @@ -1493,7 +1494,8 @@ int ACLLogMatchEntry(ACLLogEntry *a, ACLLogEntry *b) { } /* Release an ACL log entry. */ -void ACLFreeLogEntry(ACLLogEntry *le) { +void ACLFreeLogEntry(void *leptr) { + ACLLogEntry *le = leptr; sdsfree(le->object); sdsfree(le->username); sdsfree(le->cinfo); @@ -1774,7 +1776,9 @@ void aclCommand(client *c) { * the "RESET" command in order to flush the old entires. */ if (c->argc == 3) { if (!strcasecmp(c->argv[2]->ptr,"reset")) { - /* TODO: reset the log. */ + listSetFreeMethod(ACLLog,ACLFreeLogEntry); + listEmpty(ACLLog); + listSetFreeMethod(ACLLog,NULL); addReply(c,shared.ok); return; } else if (getLongFromObjectOrReply(c,c->argv[2],&count,NULL) From 82790e510f30094efecfc865ffe4c7464c50825a Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 30 Jan 2020 10:50:32 +0100 Subject: [PATCH 027/225] ACL LOG: also log ACL errors in the scripting/MULTI ctx. --- src/multi.c | 4 +++- src/scripting.c | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/multi.c b/src/multi.c index 640149870..a88e5336b 100644 --- a/src/multi.c +++ b/src/multi.c @@ -177,8 +177,10 @@ void execCommand(client *c) { must_propagate = 1; } - int acl_retval = ACLCheckCommandPerm(c,NULL); + int acl_keypos; + int acl_retval = ACLCheckCommandPerm(c,&acl_keypos); if (acl_retval != ACL_OK) { + addACLLogEntry(c,acl_retval,acl_keypos); addReplyErrorFormat(c, "-NOPERM ACLs rules changed between the moment the " "transaction was accumulated and the EXEC call. " diff --git a/src/scripting.c b/src/scripting.c index 81e174870..1007a55f7 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -606,8 +606,10 @@ int luaRedisGenericCommand(lua_State *lua, int raise_error) { } /* Check the ACLs. */ - int acl_retval = ACLCheckCommandPerm(c,NULL); + int acl_keypos; + int acl_retval = ACLCheckCommandPerm(c,&acl_keypos); if (acl_retval != ACL_OK) { + addACLLogEntry(c,acl_retval,acl_keypos); if (acl_retval == ACL_DENIED_CMD) luaPushError(lua, "The user executing the script can't run this " "command or subcommand"); From 9f6e84f6beabfc51e24580353ade795552cef8e2 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 30 Jan 2020 11:09:50 +0100 Subject: [PATCH 028/225] ACL LOG: implement a few basic tests. --- tests/unit/acl.tcl | 87 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) diff --git a/tests/unit/acl.tcl b/tests/unit/acl.tcl index 2205d2d86..bd909d36c 100644 --- a/tests/unit/acl.tcl +++ b/tests/unit/acl.tcl @@ -141,4 +141,91 @@ start_server {tags {"acl"}} { r ACL setuser newuser -debug # The test framework will detect a leak if any. } + + test {ACL LOG shows failed command executions at toplevel} { + r ACL LOG RESET + r ACL setuser antirez >foo on +set ~object:1234 + r ACL setuser antirez +eval +multi +exec + r AUTH antirez foo + catch {r GET foo} + r AUTH default "" + set entry [lindex [r ACL LOG] 0] + assert {[dict get $entry username] eq {antirez}} + assert {[dict get $entry context] eq {toplevel}} + assert {[dict get $entry reason] eq {command}} + assert {[dict get $entry object] eq {get}} + } + + test {ACL LOG is able to test similar events} { + r AUTH antirez foo + catch {r GET foo} + catch {r GET foo} + catch {r GET foo} + r AUTH default "" + set entry [lindex [r ACL LOG] 0] + assert {[dict get $entry count] == 4} + } + + test {ACL LOG is able to log keys access violations and key name} { + r AUTH antirez foo + catch {r SET somekeynotallowed 1234} + r AUTH default "" + set entry [lindex [r ACL LOG] 0] + assert {[dict get $entry reason] eq {key}} + assert {[dict get $entry object] eq {somekeynotallowed}} + } + + test {ACL LOG RESET is able to flush the entries in the log} { + r ACL LOG RESET + assert {[llength [r ACL LOG]] == 0} + } + + test {ACL LOG can distinguish the transaction context (1)} { + r AUTH antirez foo + r MULTI + catch {r INCR foo} + catch {r EXEC} + r AUTH default "" + set entry [lindex [r ACL LOG] 0] + assert {[dict get $entry context] eq {multi}} + assert {[dict get $entry object] eq {incr}} + } + + test {ACL LOG can distinguish the transaction context (2)} { + set rd1 [redis_deferring_client] + r ACL SETUSER antirez +incr + + r AUTH antirez foo + r MULTI + r INCR object:1234 + $rd1 ACL SETUSER antirez -incr + $rd1 read + catch {r EXEC} + $rd1 close + r AUTH default "" + set entry [lindex [r ACL LOG] 0] + assert {[dict get $entry context] eq {multi}} + assert {[dict get $entry object] eq {incr}} + r ACL SETUSER antirez -incr + } + + test {ACL can log errors in the context of Lua scripting} { + r AUTH antirez foo + catch {r EVAL {redis.call('incr','foo')} 0} + r AUTH default "" + set entry [lindex [r ACL LOG] 0] + assert {[dict get $entry context] eq {lua}} + assert {[dict get $entry object] eq {incr}} + } + + test {ACL LOG can accept a numerical argument to show less entries} { + r AUTH antirez foo + catch {r INCR foo} + catch {r INCR foo} + catch {r INCR foo} + catch {r INCR foo} + r AUTH default "" + assert {[llength [r ACL LOG]] > 1} + assert {[llength [r ACL LOG 2]] == 2} + } } From 7379c78a9b584ca5f394bae7ae7139ee92fa88bb Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 4 Feb 2020 12:55:26 +0100 Subject: [PATCH 029/225] ACL LOG: log failed auth attempts. --- src/acl.c | 37 +++++++++++++++++++++++++++++-------- src/multi.c | 2 +- src/scripting.c | 2 +- src/server.c | 2 +- src/server.h | 3 ++- 5 files changed, 34 insertions(+), 12 deletions(-) diff --git a/src/acl.c b/src/acl.c index 16acd4d3e..97c00d4f8 100644 --- a/src/acl.c +++ b/src/acl.c @@ -982,6 +982,7 @@ int ACLAuthenticateUser(client *c, robj *username, robj *password) { moduleNotifyUserChanged(c); return C_OK; } else { + addACLLogEntry(c,ACL_DENIED_AUTH,0,username->ptr); return C_ERR; } } @@ -1506,17 +1507,29 @@ void ACLFreeLogEntry(void *leptr) { * if we reach the maximum length allowed for the log. This function attempts * to find similar entries in the current log in order to bump the counter of * the log entry instead of creating many entries for very similar ACL - * rules issues. */ -void addACLLogEntry(client *c, int reason, int keypos) { + * rules issues. + * + * The keypos argument is only used when the reason is ACL_DENIED_KEY, since + * it allows the function to log the key name that caused the problem. + * Similarly the username is only passed when we failed to authenticate the + * user with AUTH or HELLO, for the ACL_DENIED_AUTH reason. Otherwise + * it will just be NULL. + */ +void addACLLogEntry(client *c, int reason, int keypos, sds username) { /* Create a new entry. */ struct ACLLogEntry *le = zmalloc(sizeof(*le)); le->count = 1; le->reason = reason; - le->object = (reason == ACL_DENIED_CMD) ? sdsnew(c->cmd->name) : - sdsdup(c->argv[keypos]->ptr); - le->username = sdsdup(c->user->name); + le->username = sdsdup(reason == ACL_DENIED_AUTH ? username : c->user->name); le->ctime = mstime(); + switch(reason) { + case ACL_DENIED_CMD: le->object = sdsnew(c->cmd->name); break; + case ACL_DENIED_KEY: le->object = sdsnew(c->argv[keypos]->ptr); break; + case ACL_DENIED_AUTH: le->object = sdsnew(c->argv[0]->ptr); break; + default: le->object = sdsempty(); + } + client *realclient = c; if (realclient->flags & CLIENT_LUA) realclient = server.lua_caller; @@ -1803,9 +1816,17 @@ void aclCommand(client *c) { addReplyMapLen(c,7); addReplyBulkCString(c,"count"); addReplyLongLong(c,le->count); + addReplyBulkCString(c,"reason"); - addReplyBulkCString(c,(le->reason == ACL_DENIED_CMD) ? - "command" : "key"); + char *reasonstr; + switch(le->reason) { + case ACL_DENIED_CMD: reasonstr="command"; break; + case ACL_DENIED_KEY: reasonstr="key"; break; + case ACL_DENIED_AUTH: reasonstr="auth"; break; + } + addReplyBulkCString(c,reasonstr); + + addReplyBulkCString(c,"context"); char *ctxstr; switch(le->context) { case ACL_LOG_CTX_TOPLEVEL: ctxstr="toplevel"; break; @@ -1813,8 +1834,8 @@ void aclCommand(client *c) { case ACL_LOG_CTX_LUA: ctxstr="lua"; break; default: ctxstr="unknown"; } - addReplyBulkCString(c,"context"); addReplyBulkCString(c,ctxstr); + addReplyBulkCString(c,"object"); addReplyBulkCBuffer(c,le->object,sdslen(le->object)); addReplyBulkCString(c,"username"); diff --git a/src/multi.c b/src/multi.c index a88e5336b..cbbd2c513 100644 --- a/src/multi.c +++ b/src/multi.c @@ -180,7 +180,7 @@ void execCommand(client *c) { int acl_keypos; int acl_retval = ACLCheckCommandPerm(c,&acl_keypos); if (acl_retval != ACL_OK) { - addACLLogEntry(c,acl_retval,acl_keypos); + addACLLogEntry(c,acl_retval,acl_keypos,NULL); addReplyErrorFormat(c, "-NOPERM ACLs rules changed between the moment the " "transaction was accumulated and the EXEC call. " diff --git a/src/scripting.c b/src/scripting.c index 1007a55f7..7f64e06db 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -609,7 +609,7 @@ int luaRedisGenericCommand(lua_State *lua, int raise_error) { int acl_keypos; int acl_retval = ACLCheckCommandPerm(c,&acl_keypos); if (acl_retval != ACL_OK) { - addACLLogEntry(c,acl_retval,acl_keypos); + addACLLogEntry(c,acl_retval,acl_keypos,NULL); if (acl_retval == ACL_DENIED_CMD) luaPushError(lua, "The user executing the script can't run this " "command or subcommand"); diff --git a/src/server.c b/src/server.c index 6968f311f..2827188e0 100644 --- a/src/server.c +++ b/src/server.c @@ -3380,7 +3380,7 @@ int processCommand(client *c) { int acl_keypos; int acl_retval = ACLCheckCommandPerm(c,&acl_keypos); if (acl_retval != ACL_OK) { - addACLLogEntry(c,acl_retval,acl_keypos); + addACLLogEntry(c,acl_retval,acl_keypos,NULL); flagTransaction(c); if (acl_retval == ACL_DENIED_CMD) addReplyErrorFormat(c, diff --git a/src/server.h b/src/server.h index 228a14f34..e59fac22b 100644 --- a/src/server.h +++ b/src/server.h @@ -1820,6 +1820,7 @@ void ACLInit(void); #define ACL_OK 0 #define ACL_DENIED_CMD 1 #define ACL_DENIED_KEY 2 +#define ACL_DENIED_AUTH 3 /* Only used for ACL LOG entries. */ int ACLCheckUserCredentials(robj *username, robj *password); int ACLAuthenticateUser(client *c, robj *username, robj *password); unsigned long ACLGetCommandID(const char *cmdname); @@ -1836,7 +1837,7 @@ void ACLLoadUsersAtStartup(void); void addReplyCommandCategories(client *c, struct redisCommand *cmd); user *ACLCreateUnlinkedUser(); void ACLFreeUserAndKillClients(user *u); -void addACLLogEntry(client *c, int reason, int keypos); +void addACLLogEntry(client *c, int reason, int keypos, sds username); /* Sorted sets data type */ From ea1e1b12c94c3e17707136d0d859a610260a3bea Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 4 Feb 2020 12:58:48 +0100 Subject: [PATCH 030/225] ACL LOG: test for AUTH reason. --- tests/unit/acl.tcl | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/unit/acl.tcl b/tests/unit/acl.tcl index bd909d36c..0e6d5c66a 100644 --- a/tests/unit/acl.tcl +++ b/tests/unit/acl.tcl @@ -228,4 +228,13 @@ start_server {tags {"acl"}} { assert {[llength [r ACL LOG]] > 1} assert {[llength [r ACL LOG 2]] == 2} } + + test {ACL LOG can log failed auth attempts} { + catch {r AUTH antirez wrong-password} + set entry [lindex [r ACL LOG] 0] + assert {[dict get $entry context] eq {toplevel}} + assert {[dict get $entry reason] eq {auth}} + assert {[dict get $entry object] eq {AUTH}} + assert {[dict get $entry username] eq {antirez}} + } } From 51c1a9f8fbc12a9276489178242e498bb6ccbdba Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 4 Feb 2020 13:19:40 +0100 Subject: [PATCH 031/225] ACL LOG: make max log entries configurable. --- src/acl.c | 6 ++++++ src/config.c | 1 + src/server.h | 1 + tests/unit/acl.tcl | 11 +++++++++++ 4 files changed, 19 insertions(+) diff --git a/src/acl.c b/src/acl.c index 97c00d4f8..fa57e210c 100644 --- a/src/acl.c +++ b/src/acl.c @@ -1576,6 +1576,12 @@ void addACLLogEntry(client *c, int reason, int keypos, sds username) { /* Add it to our list of entires. We'll have to trim the list * to its maximum size. */ listAddNodeHead(ACLLog, le); + while(listLength(ACLLog) > server.acllog_max_len) { + listNode *ln = listLast(ACLLog); + ACLLogEntry *le = listNodeValue(ln); + ACLFreeLogEntry(le); + listDelNode(ACLLog,ln); + } } } diff --git a/src/config.c b/src/config.c index 0526de84d..68a9b0c0d 100644 --- a/src/config.c +++ b/src/config.c @@ -2233,6 +2233,7 @@ standardConfig configs[] = { /* Unsigned Long configs */ createULongConfig("active-defrag-max-scan-fields", NULL, MODIFIABLE_CONFIG, 1, LONG_MAX, server.active_defrag_max_scan_fields, 1000, INTEGER_CONFIG, NULL, NULL), /* Default: keys with more than 1000 fields will be processed separately */ createULongConfig("slowlog-max-len", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.slowlog_max_len, 128, INTEGER_CONFIG, NULL, NULL), + createULongConfig("acllog-max-len", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.acllog_max_len, 128, INTEGER_CONFIG, NULL, NULL), /* Long Long configs */ createLongLongConfig("lua-time-limit", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.lua_time_limit, 5000, INTEGER_CONFIG, NULL, NULL),/* milliseconds */ diff --git a/src/server.h b/src/server.h index e59fac22b..4b4fa1f34 100644 --- a/src/server.h +++ b/src/server.h @@ -1385,6 +1385,7 @@ struct redisServer { dict *latency_events; /* ACLs */ char *acl_filename; /* ACL Users file. NULL if not configured. */ + unsigned long acllog_max_len; /* Maximum length of the ACL LOG list. */ /* Assert & bug reporting */ const char *assert_failed; const char *assert_file; diff --git a/tests/unit/acl.tcl b/tests/unit/acl.tcl index 0e6d5c66a..fc1664a75 100644 --- a/tests/unit/acl.tcl +++ b/tests/unit/acl.tcl @@ -237,4 +237,15 @@ start_server {tags {"acl"}} { assert {[dict get $entry object] eq {AUTH}} assert {[dict get $entry username] eq {antirez}} } + + test {ACL LOG entries are limited to a maximum amount} { + r ACL LOG RESET + r CONFIG SET acllog-max-len 5 + r AUTH antirez foo + for {set j 0} {$j < 10} {incr j} { + catch {r SET obj:$j 123} + } + r AUTH default "" + assert {[llength [r ACL LOG]] == 5} + } } From c82ccf0670465ae9ee45aa711b8f27b224aa365e Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Wed, 5 Feb 2020 09:42:49 +0200 Subject: [PATCH 032/225] memoryGetKeys helper function so that ACL can limit access to keys for MEMORY command --- src/db.c | 16 ++++++++++++++++ src/server.c | 2 +- src/server.h | 1 + 3 files changed, 18 insertions(+), 1 deletion(-) diff --git a/src/db.c b/src/db.c index ba7be2725..88342ac4d 100644 --- a/src/db.c +++ b/src/db.c @@ -1522,6 +1522,22 @@ int *georadiusGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numk return keys; } +/* Helper function to extract keys from memory command. + * MEMORY USAGE */ +int *memoryGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numkeys) { + int *keys; + UNUSED(cmd); + + if (argc >= 3 && !strcasecmp(argv[1]->ptr,"usage")) { + keys = zmalloc(sizeof(int) * 1); + keys[0] = 2; + *numkeys = 1; + return keys; + } + *numkeys = 0; + return NULL; +} + /* XREAD [BLOCK ] [COUNT ] [GROUP ] * STREAMS key_1 key_2 ... key_N ID_1 ID_2 ... ID_N */ int *xreadGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numkeys) { diff --git a/src/server.c b/src/server.c index 2827188e0..4bf20a9e2 100644 --- a/src/server.c +++ b/src/server.c @@ -817,7 +817,7 @@ struct redisCommand redisCommandTable[] = { {"memory",memoryCommand,-2, "random read-only", - 0,NULL,0,0,0,0,0,0}, + 0,memoryGetKeys,0,0,0,0,0,0}, {"client",clientCommand,-2, "admin no-script random @connection", diff --git a/src/server.h b/src/server.h index 4b4fa1f34..6a081c338 100644 --- a/src/server.h +++ b/src/server.h @@ -2077,6 +2077,7 @@ int *sortGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numkeys); int *migrateGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numkeys); int *georadiusGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numkeys); int *xreadGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numkeys); +int *memoryGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numkeys); /* Cluster */ void clusterInit(void); From 488e194787ee1e0bece37ca2d555e767e5e43372 Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Wed, 5 Feb 2020 11:41:24 +0200 Subject: [PATCH 033/225] config.c verbose error replies for CONFIG SET, like config file parsing We noticed that the error replies for the generic mechanism for enums are very verbose for config file parsing, but not for config set command. instead of replicating this code, i did a small refactoring to share code between CONFIG SET and config file parsing. and also renamed the enum group functions to be consistent with the naming of other types. --- src/config.c | 128 +++++++++++++-------------------------------------- 1 file changed, 31 insertions(+), 97 deletions(-) diff --git a/src/config.c b/src/config.c index 68a9b0c0d..d6be75795 100644 --- a/src/config.c +++ b/src/config.c @@ -190,8 +190,9 @@ typedef struct typeInterface { void (*init)(typeData data); /* Called on server start, should return 1 on success, 0 on error and should set err */ int (*load)(typeData data, sds *argc, int argv, char **err); - /* Called on CONFIG SET, returns 1 on success, 0 on error */ - int (*set)(typeData data, sds value, char **err); + /* Called on server startup and CONFIG SET, returns 1 on success, 0 on error + * and can set a verbose err string, update is true when called from CONFIG SET */ + int (*set)(typeData data, sds value, int update, char **err); /* Called on CONFIG GET, required to add output to the client */ void (*get)(client *c, typeData data); /* Called on CONFIG REWRITE, required to rewrite the config state */ @@ -323,7 +324,11 @@ void loadServerConfigFromString(char *config) { if ((!strcasecmp(argv[0],config->name) || (config->alias && !strcasecmp(argv[0],config->alias)))) { - if (!config->interface.load(config->data, argv, argc, &err)) { + if (argc != 2) { + err = "wrong number of arguments"; + goto loaderr; + } + if (!config->interface.set(config->data, argv[1], 0, &err)) { goto loaderr; } @@ -599,7 +604,7 @@ void configSetCommand(client *c) { if(config->modifiable && (!strcasecmp(c->argv[2]->ptr,config->name) || (config->alias && !strcasecmp(c->argv[2]->ptr,config->alias)))) { - if (!config->interface.set(config->data,o->ptr, &errstr)) { + if (!config->interface.set(config->data,o->ptr,1,&errstr)) { goto badfmt; } addReply(c,shared.ok); @@ -1536,9 +1541,8 @@ static char loadbuf[LOADBUF_SIZE]; .alias = (config_alias), \ .modifiable = (is_modifiable), -#define embedConfigInterface(initfn, loadfn, setfn, getfn, rewritefn) .interface = { \ +#define embedConfigInterface(initfn, setfn, getfn, rewritefn) .interface = { \ .init = (initfn), \ - .load = (loadfn), \ .set = (setfn), \ .get = (getfn), \ .rewrite = (rewritefn) \ @@ -1561,30 +1565,17 @@ static void boolConfigInit(typeData data) { *data.yesno.config = data.yesno.default_value; } -static int boolConfigLoad(typeData data, sds *argv, int argc, char **err) { - int yn; - if (argc != 2) { - *err = "wrong number of arguments"; - return 0; - } - if ((yn = yesnotoi(argv[1])) == -1) { +static int boolConfigSet(typeData data, sds value, int update, char **err) { + int yn = yesnotoi(value); + if (yn == -1) { *err = "argument must be 'yes' or 'no'"; return 0; } - if (data.yesno.is_valid_fn && !data.yesno.is_valid_fn(yn, err)) - return 0; - *data.yesno.config = yn; - return 1; -} - -static int boolConfigSet(typeData data, sds value, char **err) { - int yn = yesnotoi(value); - if (yn == -1) return 0; if (data.yesno.is_valid_fn && !data.yesno.is_valid_fn(yn, err)) return 0; int prev = *(data.yesno.config); *(data.yesno.config) = yn; - if (data.yesno.update_fn && !data.yesno.update_fn(yn, prev, err)) { + if (update && data.yesno.update_fn && !data.yesno.update_fn(yn, prev, err)) { *(data.yesno.config) = prev; return 0; } @@ -1601,7 +1592,7 @@ static void boolConfigRewrite(typeData data, const char *name, struct rewriteCon #define createBoolConfig(name, alias, modifiable, config_addr, default, is_valid, update) { \ embedCommonConfig(name, alias, modifiable) \ - embedConfigInterface(boolConfigInit, boolConfigLoad, boolConfigSet, boolConfigGet, boolConfigRewrite) \ + embedConfigInterface(boolConfigInit, boolConfigSet, boolConfigGet, boolConfigRewrite) \ .data.yesno = { \ .config = &(config_addr), \ .default_value = (default), \ @@ -1619,23 +1610,7 @@ static void stringConfigInit(typeData data) { } } -static int stringConfigLoad(typeData data, sds *argv, int argc, char **err) { - if (argc != 2) { - *err = "wrong number of arguments"; - return 0; - } - if (data.string.is_valid_fn && !data.string.is_valid_fn(argv[1], err)) - return 0; - zfree(*data.string.config); - if (data.string.convert_empty_to_null) { - *data.string.config = argv[1][0] ? zstrdup(argv[1]) : NULL; - } else { - *data.string.config = zstrdup(argv[1]); - } - return 1; -} - -static int stringConfigSet(typeData data, sds value, char **err) { +static int stringConfigSet(typeData data, sds value, int update, char **err) { if (data.string.is_valid_fn && !data.string.is_valid_fn(value, err)) return 0; char *prev = *data.string.config; @@ -1644,7 +1619,7 @@ static int stringConfigSet(typeData data, sds value, char **err) { } else { *data.string.config = zstrdup(value); } - if (data.string.update_fn && !data.string.update_fn(*data.string.config, prev, err)) { + if (update && data.string.update_fn && !data.string.update_fn(*data.string.config, prev, err)) { zfree(*data.string.config); *data.string.config = prev; return 0; @@ -1666,7 +1641,7 @@ static void stringConfigRewrite(typeData data, const char *name, struct rewriteC #define createStringConfig(name, alias, modifiable, empty_to_null, config_addr, default, is_valid, update) { \ embedCommonConfig(name, alias, modifiable) \ - embedConfigInterface(stringConfigInit, stringConfigLoad, stringConfigSet, stringConfigGet, stringConfigRewrite) \ + embedConfigInterface(stringConfigInit, stringConfigSet, stringConfigGet, stringConfigRewrite) \ .data.string = { \ .config = &(config_addr), \ .default_value = (default), \ @@ -1677,17 +1652,12 @@ static void stringConfigRewrite(typeData data, const char *name, struct rewriteC } /* Enum configs */ -static void configEnumInit(typeData data) { +static void enumConfigInit(typeData data) { *data.enumd.config = data.enumd.default_value; } -static int configEnumLoad(typeData data, sds *argv, int argc, char **err) { - if (argc != 2) { - *err = "wrong number of arguments"; - return 0; - } - - int enumval = configEnumGetValue(data.enumd.enum_value, argv[1]); +static int enumConfigSet(typeData data, sds value, int update, char **err) { + int enumval = configEnumGetValue(data.enumd.enum_value, value); if (enumval == INT_MIN) { sds enumerr = sdsnew("argument must be one of the following: "); configEnum *enumNode = data.enumd.enum_value; @@ -1707,37 +1677,28 @@ static int configEnumLoad(typeData data, sds *argv, int argc, char **err) { *err = loadbuf; return 0; } - if (data.enumd.is_valid_fn && !data.enumd.is_valid_fn(enumval, err)) - return 0; - *(data.enumd.config) = enumval; - return 1; -} - -static int configEnumSet(typeData data, sds value, char **err) { - int enumval = configEnumGetValue(data.enumd.enum_value, value); - if (enumval == INT_MIN) return 0; if (data.enumd.is_valid_fn && !data.enumd.is_valid_fn(enumval, err)) return 0; int prev = *(data.enumd.config); *(data.enumd.config) = enumval; - if (data.enumd.update_fn && !data.enumd.update_fn(enumval, prev, err)) { + if (update && data.enumd.update_fn && !data.enumd.update_fn(enumval, prev, err)) { *(data.enumd.config) = prev; return 0; } return 1; } -static void configEnumGet(client *c, typeData data) { +static void enumConfigGet(client *c, typeData data) { addReplyBulkCString(c, configEnumGetNameOrUnknown(data.enumd.enum_value,*data.enumd.config)); } -static void configEnumRewrite(typeData data, const char *name, struct rewriteConfigState *state) { +static void enumConfigRewrite(typeData data, const char *name, struct rewriteConfigState *state) { rewriteConfigEnumOption(state, name,*(data.enumd.config), data.enumd.enum_value, data.enumd.default_value); } #define createEnumConfig(name, alias, modifiable, enum, config_addr, default, is_valid, update) { \ embedCommonConfig(name, alias, modifiable) \ - embedConfigInterface(configEnumInit, configEnumLoad, configEnumSet, configEnumGet, configEnumRewrite) \ + embedConfigInterface(enumConfigInit, enumConfigSet, enumConfigGet, enumConfigRewrite) \ .data.enumd = { \ .config = &(config_addr), \ .default_value = (default), \ @@ -1832,49 +1793,22 @@ static int numericBoundaryCheck(typeData data, long long ll, char **err) { return 1; } -static int numericConfigLoad(typeData data, sds *argv, int argc, char **err) { - long long ll; - - if (argc != 2) { - *err = "wrong number of arguments"; - return 0; - } - +static int numericConfigSet(typeData data, sds value, int update, char **err) { + long long ll, prev = 0; if (data.numeric.is_memory) { int memerr; - ll = memtoll(argv[1], &memerr); + ll = memtoll(value, &memerr); if (memerr || ll < 0) { *err = "argument must be a memory value"; return 0; } } else { - if (!string2ll(argv[1], sdslen(argv[1]),&ll)) { + if (!string2ll(value, sdslen(value),&ll)) { *err = "argument couldn't be parsed into an integer" ; return 0; } } - if (!numericBoundaryCheck(data, ll, err)) - return 0; - - if (data.numeric.is_valid_fn && !data.numeric.is_valid_fn(ll, err)) - return 0; - - SET_NUMERIC_TYPE(ll) - - return 1; -} - -static int numericConfigSet(typeData data, sds value, char **err) { - long long ll, prev = 0; - if (data.numeric.is_memory) { - int memerr; - ll = memtoll(value, &memerr); - if (memerr || ll < 0) return 0; - } else { - if (!string2ll(value, sdslen(value),&ll)) return 0; - } - if (!numericBoundaryCheck(data, ll, err)) return 0; @@ -1884,7 +1818,7 @@ static int numericConfigSet(typeData data, sds value, char **err) { GET_NUMERIC_TYPE(prev) SET_NUMERIC_TYPE(ll) - if (data.numeric.update_fn && !data.numeric.update_fn(ll, prev, err)) { + if (update && data.numeric.update_fn && !data.numeric.update_fn(ll, prev, err)) { SET_NUMERIC_TYPE(prev) return 0; } @@ -1918,7 +1852,7 @@ static void numericConfigRewrite(typeData data, const char *name, struct rewrite #define embedCommonNumericalConfig(name, alias, modifiable, lower, upper, config_addr, default, memory, is_valid, update) { \ embedCommonConfig(name, alias, modifiable) \ - embedConfigInterface(numericConfigInit, numericConfigLoad, numericConfigSet, numericConfigGet, numericConfigRewrite) \ + embedConfigInterface(numericConfigInit, numericConfigSet, numericConfigGet, numericConfigRewrite) \ .data.numeric = { \ .lower_bound = (lower), \ .upper_bound = (upper), \ From 736309660f7531c059690184c17798010b8155e8 Mon Sep 17 00:00:00 2001 From: Yossi Gottlieb Date: Wed, 5 Feb 2020 18:30:12 +0200 Subject: [PATCH 034/225] TLS: Some redis.conf clarifications. --- redis.conf | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/redis.conf b/redis.conf index 07005cffe..3c7336747 100644 --- a/redis.conf +++ b/redis.conf @@ -155,23 +155,22 @@ tcp-keepalive 300 # tls-ca-cert-file ca.crt # tls-ca-cert-dir /etc/ssl/certs -# If TLS/SSL clients are required to authenticate using a client side -# certificate, use this directive. +# By default, clients (including replica servers) on a TLS port are required +# to authenticate using valid client side certificates. # -# Note: this applies to all incoming clients, including replicas. +# It is possible to disable authentication using this directive. # -# tls-auth-clients yes +# tls-auth-clients no -# If TLS/SSL should be used when connecting as a replica to a master, enable -# this configuration directive: +# By default, a Redis replica does not attempt to establish a TLS connection +# with its master. +# +# Use the following directive to enable TLS on replication links. # # tls-replication yes -# If TLS/SSL should be used for the Redis Cluster bus, enable this configuration -# directive. -# -# NOTE: If TLS/SSL is enabled for Cluster Bus, mutual authentication is always -# enforced. +# By default, the Redis Cluster bus uses a plain TCP connection. To enable +# TLS for the bus protocol, use the following directive: # # tls-cluster yes From fae306b3745556458a9f4900cdcbbce4affbcb71 Mon Sep 17 00:00:00 2001 From: Guy Benoish Date: Thu, 6 Feb 2020 14:09:45 +0530 Subject: [PATCH 035/225] Fix small bugs related to replica and monitor ambiguity 1. server.repl_no_slaves_since can be set when a MONITOR client disconnects 2. c->repl_ack_time can be set by a newline from a MONITOR client 3. Improved comments --- src/networking.c | 12 +++++++----- src/server.c | 2 +- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/networking.c b/src/networking.c index a2e454d4b..8fca3fc1d 100644 --- a/src/networking.c +++ b/src/networking.c @@ -1121,7 +1121,7 @@ void freeClient(client *c) { /* We need to remember the time when we started to have zero * attached slaves, as after some time we'll free the replication * backlog. */ - if (c->flags & CLIENT_SLAVE && listLength(server.slaves) == 0) + if (getClientType(c) == CLIENT_TYPE_SLAVE && listLength(server.slaves) == 0) server.repl_no_slaves_since = server.unixtime; refreshGoodSlavesCount(); /* Fire the replica change modules event. */ @@ -1252,8 +1252,8 @@ int writeToClient(client *c, int handler_installed) { * just deliver as much data as it is possible to deliver. * * Moreover, we also send as much as possible if the client is - * a slave (otherwise, on high-speed traffic, the replication - * buffer will grow indefinitely) */ + * a slave or a monitor (otherwise, on high-speed traffic, the + * replication/output buffer will grow indefinitely) */ if (totwritten > NET_MAX_WRITES_PER_EVENT && (server.maxmemory == 0 || zmalloc_used_memory() < server.maxmemory) && @@ -1439,7 +1439,7 @@ int processInlineBuffer(client *c) { /* Newline from slaves can be used to refresh the last ACK time. * This is useful for a slave to ping back while loading a big * RDB file. */ - if (querylen == 0 && c->flags & CLIENT_SLAVE) + if (querylen == 0 && getClientType(c) == CLIENT_TYPE_SLAVE) c->repl_ack_time = server.unixtime; /* Move querybuffer position to the next query in the buffer. */ @@ -2433,12 +2433,14 @@ unsigned long getClientOutputBufferMemoryUsage(client *c) { * * The function will return one of the following: * CLIENT_TYPE_NORMAL -> Normal client - * CLIENT_TYPE_SLAVE -> Slave or client executing MONITOR command + * CLIENT_TYPE_SLAVE -> Slave * CLIENT_TYPE_PUBSUB -> Client subscribed to Pub/Sub channels * CLIENT_TYPE_MASTER -> The client representing our replication master. */ int getClientType(client *c) { if (c->flags & CLIENT_MASTER) return CLIENT_TYPE_MASTER; + /* Even though MONITOR clients are marked as replicas, we + * want the expose them as normal clients. */ if ((c->flags & CLIENT_SLAVE) && !(c->flags & CLIENT_MONITOR)) return CLIENT_TYPE_SLAVE; if (c->flags & CLIENT_PUBSUB) return CLIENT_TYPE_PUBSUB; diff --git a/src/server.c b/src/server.c index 4bf20a9e2..1a3da249f 100644 --- a/src/server.c +++ b/src/server.c @@ -1498,7 +1498,7 @@ int clientsCronHandleTimeout(client *c, mstime_t now_ms) { time_t now = now_ms/1000; if (server.maxidletime && - !(c->flags & CLIENT_SLAVE) && /* no timeout for slaves */ + !(c->flags & CLIENT_SLAVE) && /* no timeout for slaves and monitors */ !(c->flags & CLIENT_MASTER) && /* no timeout for masters */ !(c->flags & CLIENT_BLOCKED) && /* no timeout for BLPOP */ !(c->flags & CLIENT_PUBSUB) && /* no timeout for Pub/Sub clients */ From 5a6cfbf4ca737be71178624ef721e6dd053fb645 Mon Sep 17 00:00:00 2001 From: Guy Benoish Date: Thu, 6 Feb 2020 14:12:08 +0530 Subject: [PATCH 036/225] Some refactroing using getClientType instead of CLIENT_SLAVE --- src/networking.c | 9 +++++---- src/object.c | 35 +++++++++++++---------------------- 2 files changed, 18 insertions(+), 26 deletions(-) diff --git a/src/networking.c b/src/networking.c index 8fca3fc1d..496b0f3dc 100644 --- a/src/networking.c +++ b/src/networking.c @@ -369,9 +369,10 @@ void addReplyErrorLength(client *c, const char *s, size_t len) { * Where the master must propagate the first change even if the second * will produce an error. However it is useful to log such events since * they are rare and may hint at errors in a script or a bug in Redis. */ - if (c->flags & (CLIENT_MASTER|CLIENT_SLAVE) && !(c->flags & CLIENT_MONITOR)) { - char* to = c->flags & CLIENT_MASTER? "master": "replica"; - char* from = c->flags & CLIENT_MASTER? "replica": "master"; + int ctype = getClientType(c); + if (ctype == CLIENT_TYPE_MASTER || ctype == CLIENT_TYPE_SLAVE) { + char* to = ctype == CLIENT_TYPE_MASTER? "master": "replica"; + char* from = ctype == CLIENT_TYPE_MASTER? "replica": "master"; char *cmdname = c->lastcmd ? c->lastcmd->name : ""; serverLog(LL_WARNING,"== CRITICAL == This %s is sending an error " "to its %s: '%s' after processing the command " @@ -1074,7 +1075,7 @@ void freeClient(client *c) { } /* Log link disconnection with slave */ - if ((c->flags & CLIENT_SLAVE) && !(c->flags & CLIENT_MONITOR)) { + if (getClientType(c) == CLIENT_TYPE_SLAVE) { serverLog(LL_WARNING,"Connection with replica %s lost.", replicationGetSlaveName(c)); } diff --git a/src/object.c b/src/object.c index 52007b474..cc6b218a0 100644 --- a/src/object.c +++ b/src/object.c @@ -974,38 +974,29 @@ struct redisMemOverhead *getMemoryOverheadData(void) { mh->repl_backlog = mem; mem_total += mem; - mem = 0; - if (listLength(server.slaves)) { - listIter li; - listNode *ln; - - listRewind(server.slaves,&li); - while((ln = listNext(&li))) { - client *c = listNodeValue(ln); - mem += getClientOutputBufferMemoryUsage(c); - mem += sdsAllocSize(c->querybuf); - mem += sizeof(client); - } - } - mh->clients_slaves = mem; - mem_total+=mem; - mem = 0; if (listLength(server.clients)) { listIter li; listNode *ln; + size_t mem_normal = 0, mem_slaves = 0; listRewind(server.clients,&li); while((ln = listNext(&li))) { + size_t mem_curr = 0; client *c = listNodeValue(ln); - if (c->flags & CLIENT_SLAVE && !(c->flags & CLIENT_MONITOR)) - continue; - mem += getClientOutputBufferMemoryUsage(c); - mem += sdsAllocSize(c->querybuf); - mem += sizeof(client); + int type = getClientType(c); + mem_curr += getClientOutputBufferMemoryUsage(c); + mem_curr += sdsAllocSize(c->querybuf); + mem_curr += sizeof(client); + if (type == CLIENT_TYPE_SLAVE) + mem_slaves += mem_curr; + else + mem_normal += mem_curr; } + mh->clients_slaves = mem_slaves; + mh->clients_normal = mem_normal; + mem = mem_slaves + mem_normal; } - mh->clients_normal = mem; mem_total+=mem; mem = 0; From ba289244185fd717c4ebc52ae6ab02d9b7bcdd25 Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Thu, 6 Feb 2020 10:14:32 +0200 Subject: [PATCH 037/225] move restartAOFAfterSYNC from replicaofCommand to replicationUnsetMaster replicationUnsetMaster can be called from other places, not just replicaofCOmmand, and all of these need to restart AOF --- src/replication.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/replication.c b/src/replication.c index b7e77184a..5499ebc57 100644 --- a/src/replication.c +++ b/src/replication.c @@ -2399,6 +2399,10 @@ void replicationUnsetMaster(void) { moduleFireServerEvent(REDISMODULE_EVENT_REPLICATION_ROLE_CHANGED, REDISMODULE_EVENT_REPLROLECHANGED_NOW_MASTER, NULL); + + /* Restart the AOF subsystem in case we shut it down during a sync when + * we were still a slave. */ + if (server.aof_enabled && server.aof_state == AOF_OFF) restartAOFAfterSYNC(); } /* This function is called when the slave lose the connection with the @@ -2436,9 +2440,6 @@ void replicaofCommand(client *c) { serverLog(LL_NOTICE,"MASTER MODE enabled (user request from '%s')", client); sdsfree(client); - /* Restart the AOF subsystem in case we shut it down during a sync when - * we were still a slave. */ - if (server.aof_enabled && server.aof_state == AOF_OFF) restartAOFAfterSYNC(); } } else { long port; From a55e5847067a6c0a68371756b290d77f8f91d25d Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Thu, 6 Feb 2020 10:40:29 +0200 Subject: [PATCH 038/225] DEBUG HELP - add PROTOCOL --- src/debug.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/debug.c b/src/debug.c index a2d37337d..b910d2d2d 100644 --- a/src/debug.c +++ b/src/debug.c @@ -355,6 +355,7 @@ void debugCommand(client *c) { "CRASH-AND-RECOVER -- Hard crash and restart after delay.", "DIGEST -- Output a hex signature representing the current DB content.", "DIGEST-VALUE ... -- Output a hex signature of the values of all the specified keys.", +"DEBUG PROTOCOL [string|integer|double|bignum|null|array|set|map|attrib|push|verbatim|true|false]", "ERROR -- Return a Redis protocol error with as message. Useful for clients unit tests to simulate Redis errors.", "LOG -- write message to the server log.", "HTSTATS -- Return hash table statistics of the specified Redis database.", @@ -586,7 +587,7 @@ NULL } } else if (!strcasecmp(c->argv[1]->ptr,"protocol") && c->argc == 3) { /* DEBUG PROTOCOL [string|integer|double|bignum|null|array|set|map| - * attrib|push|verbatim|true|false|state|err|bloberr] */ + * attrib|push|verbatim|true|false] */ char *name = c->argv[2]->ptr; if (!strcasecmp(name,"string")) { addReplyBulkCString(c,"Hello World"); @@ -634,7 +635,7 @@ NULL } else if (!strcasecmp(name,"verbatim")) { addReplyVerbatim(c,"This is a verbatim\nstring",25,"txt"); } else { - addReplyError(c,"Wrong protocol type name. Please use one of the following: string|integer|double|bignum|null|array|set|map|attrib|push|verbatim|true|false|state|err|bloberr"); + addReplyError(c,"Wrong protocol type name. Please use one of the following: string|integer|double|bignum|null|array|set|map|attrib|push|verbatim|true|false"); } } else if (!strcasecmp(c->argv[1]->ptr,"sleep") && c->argc == 3) { double dtime = strtod(c->argv[2]->ptr,NULL); From 22e45d46fe8aeb1fa385fdca5db80eac6776a290 Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Thu, 6 Feb 2020 10:07:17 +0200 Subject: [PATCH 039/225] freeClientAsync don't lock mutex if there's just one thread --- src/networking.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/networking.c b/src/networking.c index 496b0f3dc..2b0f7464a 100644 --- a/src/networking.c +++ b/src/networking.c @@ -1163,9 +1163,14 @@ void freeClientAsync(client *c) { * may access the list while Redis uses I/O threads. All the other accesses * are in the context of the main thread while the other threads are * idle. */ - static pthread_mutex_t async_free_queue_mutex = PTHREAD_MUTEX_INITIALIZER; if (c->flags & CLIENT_CLOSE_ASAP || c->flags & CLIENT_LUA) return; c->flags |= CLIENT_CLOSE_ASAP; + if (server.io_threads_num == 1) { + /* no need to bother with locking if there's just one thread (the main thread) */ + listAddNodeTail(server.clients_to_close,c); + return; + } + static pthread_mutex_t async_free_queue_mutex = PTHREAD_MUTEX_INITIALIZER; pthread_mutex_lock(&async_free_queue_mutex); listAddNodeTail(server.clients_to_close,c); pthread_mutex_unlock(&async_free_queue_mutex); From df096bc96bad620a89797bf94298f53a227ff76b Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Thu, 6 Feb 2020 10:31:43 +0200 Subject: [PATCH 040/225] add SAVE subcommand to ACL HELP and top comment --- src/acl.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/acl.c b/src/acl.c index fa57e210c..b046785ff 100644 --- a/src/acl.c +++ b/src/acl.c @@ -1592,6 +1592,7 @@ void addACLLogEntry(client *c, int reason, int keypos, sds username) { /* ACL -- show and modify the configuration of ACL users. * ACL HELP * ACL LOAD + * ACL SAVE * ACL LIST * ACL USERS * ACL CAT [] @@ -1855,6 +1856,7 @@ void aclCommand(client *c) { } else if (!strcasecmp(sub,"help")) { const char *help[] = { "LOAD -- Reload users from the ACL file.", +"SAVE -- Save the current config to the ACL file." "LIST -- Show user details in config file format.", "USERS -- List all the registered usernames.", "SETUSER [attribs ...] -- Create or modify a user.", From 919fbf421d0a1aed5648d6df82ccca99105e6855 Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Thu, 6 Feb 2020 09:41:45 +0200 Subject: [PATCH 041/225] reduce repeated calls to use_diskless_load this function possibly iterates on the module list --- src/replication.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/replication.c b/src/replication.c index 5499ebc57..a006d0ad1 100644 --- a/src/replication.c +++ b/src/replication.c @@ -1354,7 +1354,7 @@ void disklessLoadRestoreBackups(redisDb *backup, int restore, int empty_db_flags void readSyncBulkPayload(connection *conn) { char buf[4096]; ssize_t nread, readlen, nwritten; - int use_diskless_load; + int use_diskless_load = useDisklessLoad(); redisDb *diskless_load_backup = NULL; int empty_db_flags = server.repl_slave_lazy_flush ? EMPTYDB_ASYNC : EMPTYDB_NO_FLAGS; @@ -1411,19 +1411,18 @@ void readSyncBulkPayload(connection *conn) { server.repl_transfer_size = 0; serverLog(LL_NOTICE, "MASTER <-> REPLICA sync: receiving streamed RDB from master with EOF %s", - useDisklessLoad()? "to parser":"to disk"); + use_diskless_load? "to parser":"to disk"); } else { usemark = 0; server.repl_transfer_size = strtol(buf+1,NULL,10); serverLog(LL_NOTICE, "MASTER <-> REPLICA sync: receiving %lld bytes from master %s", (long long) server.repl_transfer_size, - useDisklessLoad()? "to parser":"to disk"); + use_diskless_load? "to parser":"to disk"); } return; } - use_diskless_load = useDisklessLoad(); if (!use_diskless_load) { /* Read the data from the socket, store it to a file and search * for the EOF. */ From f42ce57d0f7eee6705fb0a81714cabe97a4f2c0a Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Thu, 6 Feb 2020 10:17:34 +0200 Subject: [PATCH 042/225] stopAppendOnly resets aof_rewrite_scheduled althouh in theory, users can do BGREWRITEAOF even if aof is disabled, i suppose it is more common that the scheduled flag is set by either startAppendOnly, of a failed initial AOFRW fork (AOF_WAIT_REWRITE) --- src/aof.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/aof.c b/src/aof.c index 9eeb3f1e2..3682c4568 100644 --- a/src/aof.c +++ b/src/aof.c @@ -242,6 +242,7 @@ void stopAppendOnly(void) { server.aof_fd = -1; server.aof_selected_db = -1; server.aof_state = AOF_OFF; + server.aof_rewrite_scheduled = 0; killAppendOnlyChild(); } From a678390e52cf0db89486fcecc47e8114449bb5dd Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Thu, 6 Feb 2020 09:37:04 +0200 Subject: [PATCH 043/225] moduleRDBLoadError, add key name, and use panic rather than exit using panic rather than exit means you get s stack trace of the code path that experianced the error, and possibly other info. --- src/module.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/module.c b/src/module.c index 914c50df3..16b191535 100644 --- a/src/module.c +++ b/src/module.c @@ -3649,14 +3649,15 @@ void moduleRDBLoadError(RedisModuleIO *io) { io->error = 1; return; } - serverLog(LL_WARNING, + serverPanic( "Error loading data from RDB (short read or EOF). " "Read performed by module '%s' about type '%s' " - "after reading '%llu' bytes of a value.", + "after reading '%llu' bytes of a value " + "for key named: '%s'.", io->type->module->name, io->type->name, - (unsigned long long)io->bytes); - exit(1); + (unsigned long long)io->bytes, + io->key? (char*)io->key->ptr: "(null)"); } /* Returns 0 if there's at least one registered data type that did not declare From 8e7282eb3ec17052cb55b597036a02d0c4200cdf Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Thu, 6 Feb 2020 09:33:20 +0200 Subject: [PATCH 044/225] Fix client flags to be int64 in module.c currently there's no bug since the flags these functions handle are always lower than 32bit, but still better fix the type to prevent future bugs. --- src/module.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/module.c b/src/module.c index 16b191535..3516b8f13 100644 --- a/src/module.c +++ b/src/module.c @@ -714,9 +714,9 @@ void RM_KeyAtPos(RedisModuleCtx *ctx, int pos) { * flags into the command flags used by the Redis core. * * It returns the set of flags, or -1 if unknown flags are found. */ -int commandFlagsFromString(char *s) { +int64_t commandFlagsFromString(char *s) { int count, j; - int flags = 0; + int64_t flags = 0; sds *tokens = sdssplitlen(s,strlen(s)," ",1,&count); for (j = 0; j < count; j++) { char *t = tokens[j]; @@ -798,7 +798,7 @@ int commandFlagsFromString(char *s) { * to authenticate a client. */ int RM_CreateCommand(RedisModuleCtx *ctx, const char *name, RedisModuleCmdFunc cmdfunc, const char *strflags, int firstkey, int lastkey, int keystep) { - int flags = strflags ? commandFlagsFromString((char*)strflags) : 0; + int64_t flags = strflags ? commandFlagsFromString((char*)strflags) : 0; if (flags == -1) return REDISMODULE_ERR; if ((flags & CMD_MODULE_NO_CLUSTER) && server.cluster_enabled) return REDISMODULE_ERR; From 1333a46b7e45395cb47ddde760ab8de3833d9f73 Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Thu, 6 Feb 2020 09:23:22 +0200 Subject: [PATCH 045/225] fix maxmemory config warning the warning condition was if usage > limit (saying it'll cause eviction or oom), but in fact the eviction and oom depends on used minus slave buffers. other than fixing the condition, i add info about the current usage and limit, which may be useful when looking at the log. --- src/config.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/config.c b/src/config.c index d6be75795..1e8d0ffe2 100644 --- a/src/config.c +++ b/src/config.c @@ -1995,8 +1995,9 @@ static int updateMaxmemory(long long val, long long prev, char **err) { UNUSED(prev); UNUSED(err); if (val) { - if ((unsigned long long)val < zmalloc_used_memory()) { - serverLog(LL_WARNING,"WARNING: the new maxmemory value set via CONFIG SET is smaller than the current memory usage. This will result in key eviction and/or the inability to accept new write commands depending on the maxmemory-policy."); + size_t used = zmalloc_used_memory()-freeMemoryGetNotCountedMemory(); + if ((unsigned long long)val < used) { + serverLog(LL_WARNING,"WARNING: the new maxmemory value set via CONFIG SET (%llu) is smaller than the current memory usage (%zu). This will result in key eviction and/or the inability to accept new write commands depending on the maxmemory-policy.", server.maxmemory, used); } freeMemoryIfNeededAndSafe(); } From c9577941442618e257c891bb1c382d2612d88c4f Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Thu, 6 Feb 2020 09:15:31 +0200 Subject: [PATCH 046/225] Memory leak when bind config is provided twice --- src/config.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/config.c b/src/config.c index 1e8d0ffe2..a6b374817 100644 --- a/src/config.c +++ b/src/config.c @@ -349,6 +349,10 @@ void loadServerConfigFromString(char *config) { if (addresses > CONFIG_BINDADDR_MAX) { err = "Too many bind addresses specified"; goto loaderr; } + /* Free old bind addresses */ + for (j = 0; j < server.bindaddr_count; j++) { + zfree(server.bindaddr[j]); + } for (j = 0; j < addresses; j++) server.bindaddr[j] = zstrdup(argv[j+1]); server.bindaddr_count = addresses; From 4440133e9ac043e9f1db14d8630dfb08eaade9b2 Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Thu, 6 Feb 2020 08:53:23 +0200 Subject: [PATCH 047/225] A few non-data commands that should be allowed while loading or stale SELECT, and HELLO are commands that may be executed by the client as soon as it connects, there's no reason to block them, preventing the client from doing the rest of his sequence (which might just be INFO or CONFIG, etc). MONITOR, DEBUG, SLOWLOG, TIME, LASTSAVE are all non-data accessing commands, which there's no reason to block. --- src/server.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/server.c b/src/server.c index 1a3da249f..f87dfba46 100644 --- a/src/server.c +++ b/src/server.c @@ -579,7 +579,7 @@ struct redisCommand redisCommandTable[] = { 0,NULL,0,0,0,0,0,0}, {"select",selectCommand,2, - "ok-loading fast @keyspace", + "ok-loading fast ok-stale @keyspace", 0,NULL,0,0,0,0,0,0}, {"swapdb",swapdbCommand,3, @@ -660,7 +660,7 @@ struct redisCommand redisCommandTable[] = { 0,NULL,0,0,0,0,0,0}, {"lastsave",lastsaveCommand,1, - "read-only random fast @admin @dangerous", + "read-only random fast ok-loading ok-stale @admin @dangerous", 0,NULL,0,0,0,0,0,0}, {"type",typeCommand,2, @@ -708,7 +708,7 @@ struct redisCommand redisCommandTable[] = { 0,NULL,0,0,0,0,0,0}, {"monitor",monitorCommand,1, - "admin no-script", + "admin no-script ok-loading ok-stale", 0,NULL,0,0,0,0,0,0}, {"ttl",ttlCommand,2, @@ -740,7 +740,7 @@ struct redisCommand redisCommandTable[] = { 0,NULL,0,0,0,0,0,0}, {"debug",debugCommand,-2, - "admin no-script", + "admin no-script ok-loading ok-stale", 0,NULL,0,0,0,0,0,0}, {"config",configCommand,-2, @@ -820,11 +820,11 @@ struct redisCommand redisCommandTable[] = { 0,memoryGetKeys,0,0,0,0,0,0}, {"client",clientCommand,-2, - "admin no-script random @connection", + "admin no-script random ok-loading ok-stale @connection", 0,NULL,0,0,0,0,0,0}, {"hello",helloCommand,-2, - "no-auth no-script fast no-monitor no-slowlog @connection", + "no-auth no-script fast no-monitor ok-loading ok-stale no-slowlog @connection", 0,NULL,0,0,0,0,0,0}, /* EVAL can modify the dataset, however it is not flagged as a write @@ -838,7 +838,7 @@ struct redisCommand redisCommandTable[] = { 0,evalGetKeys,0,0,0,0,0,0}, {"slowlog",slowlogCommand,-2, - "admin random", + "admin random ok-loading ok-stale", 0,NULL,0,0,0,0,0,0}, {"script",scriptCommand,-2, @@ -846,7 +846,7 @@ struct redisCommand redisCommandTable[] = { 0,NULL,0,0,0,0,0,0}, {"time",timeCommand,1, - "read-only random fast", + "read-only random fast ok-loading ok-stale", 0,NULL,0,0,0,0,0,0}, {"bitop",bitopCommand,-4, From 9baaf858f940fbb5b5a8e90f37d5b6ae15ceb9ee Mon Sep 17 00:00:00 2001 From: Yossi Gottlieb Date: Wed, 5 Feb 2020 21:13:21 +0200 Subject: [PATCH 048/225] TLS: Update documentation. --- README.md | 18 ++++++++++++++++++ TLS.md | 45 ++++++++++++++------------------------------- 2 files changed, 32 insertions(+), 31 deletions(-) diff --git a/README.md b/README.md index 3442659e6..c08013416 100644 --- a/README.md +++ b/README.md @@ -35,6 +35,11 @@ It is as simple as: % make +To build with TLS support, you'll need OpenSSL development libraries (e.g. +libssl-dev on Debian/Ubuntu) and run: + + % make BUILD_TLS=yes + You can run a 32 bit Redis binary using: % make 32bit @@ -43,6 +48,13 @@ After building Redis, it is a good idea to test it using: % make test +If TLS is built, running the tests with TLS enabled (you will need `tcl-tls` +installed): + + % ./utils/gen-test-certs.sh + % ./runtest --tls + + Fixing build problems with dependencies or cached build options --------- @@ -125,6 +137,12 @@ as options using the command line. Examples: All the options in redis.conf are also supported as options using the command line, with exactly the same name. +Running Redis with TLS: +------------------ + +Please consult the [TLS.md](TLS.md) file for more information on +how to use Redis with TLS. + Playing with Redis ------------------ diff --git a/TLS.md b/TLS.md index 76fe0be2e..e480c1e9d 100644 --- a/TLS.md +++ b/TLS.md @@ -1,8 +1,5 @@ -TLS Support -- Work In Progress -=============================== - -This is a brief note to capture current thoughts/ideas and track pending action -items. +TLS Support +=========== Getting Started --------------- @@ -69,37 +66,23 @@ probably not be so hard. For cluster keys migration it might be more difficult, but there are probably other good reasons to improve that part anyway. To-Do List -========== +---------- -Additional TLS Features ------------------------ +- [ ] Add session caching support. Check if/how it's handled by clients to + assess how useful/important it is. +- [ ] redis-benchmark support. The current implementation is a mix of using + hiredis for parsing and basic networking (establishing connections), but + directly manipulating sockets for most actions. This will need to be cleaned + up for proper TLS support. The best approach is probably to migrate to hiredis + async mode. +- [ ] redis-cli `--slave` and `--rdb` support. -1. Add metrics to INFO? -2. Add session caching support. Check if/how it's handled by clients to assess - how useful/important it is. - -redis-benchmark ---------------- - -The current implementation is a mix of using hiredis for parsing and basic -networking (establishing connections), but directly manipulating sockets for -most actions. - -This will need to be cleaned up for proper TLS support. The best approach is -probably to migrate to hiredis async mode. - -redis-cli ---------- - -1. Add support for TLS in --slave and --rdb modes. - -Others ------- +Multi-port +---------- Consider the implications of allowing TLS to be configured on a separate port, -making Redis listening on multiple ports. +making Redis listening on multiple ports: -This impacts many things, like 1. Startup banner port notification 2. Proctitle 3. How slaves announce themselves From 3067352a8d8883b459e3029c4edb9e536b207d48 Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Wed, 5 Feb 2020 19:47:09 +0200 Subject: [PATCH 049/225] Add handling of short read of module id in rdb --- src/rdb.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/rdb.c b/src/rdb.c index 27e1b3135..61265433d 100644 --- a/src/rdb.c +++ b/src/rdb.c @@ -1871,7 +1871,10 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, robj *key) { } } else if (rdbtype == RDB_TYPE_MODULE || rdbtype == RDB_TYPE_MODULE_2) { uint64_t moduleid = rdbLoadLen(rdb,NULL); - if (rioGetReadError(rdb)) return NULL; + if (rioGetReadError(rdb)) { + rdbReportReadError("Short read module id"); + return NULL; + } moduleType *mt = moduleTypeLookupModuleByID(moduleid); char name[10]; From 36caf2e42b2ad4174af0760a597bc34d6e2569bd Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Wed, 5 Feb 2020 18:15:38 +0200 Subject: [PATCH 050/225] update RM_SignalModifiedKey doc comment --- src/module.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/module.c b/src/module.c index 3516b8f13..b39383637 100644 --- a/src/module.c +++ b/src/module.c @@ -890,7 +890,8 @@ void RM_SetModuleOptions(RedisModuleCtx *ctx, int options) { ctx->module->options = options; } -/* Signals that the key is modified from user's perspective (i.e. invalidate WATCH). */ +/* Signals that the key is modified from user's perspective (i.e. invalidate WATCH + * and client side caching). */ int RM_SignalModifiedKey(RedisModuleCtx *ctx, RedisModuleString *keyname) { signalModifiedKey(ctx->client->db,keyname); return REDISMODULE_OK; From dcbe8bfad143b32230125710756285be5179722b Mon Sep 17 00:00:00 2001 From: Guy Benoish Date: Mon, 3 Feb 2020 15:19:44 +0530 Subject: [PATCH 051/225] Exclude "keymiss" notification from NOTIFY_ALL Because "keymiss" is "special" compared to the rest of the notifications (Trying not to break existing apps using the 'A' format for notifications) Also updated redis.conf and module.c docs --- redis.conf | 6 +++++- src/module.c | 3 ++- src/notify.c | 2 +- src/redismodule.h | 4 ++-- src/server.h | 4 ++-- 5 files changed, 12 insertions(+), 7 deletions(-) diff --git a/redis.conf b/redis.conf index 3c7336747..c04880f32 100644 --- a/redis.conf +++ b/redis.conf @@ -1361,7 +1361,11 @@ latency-monitor-threshold 0 # z Sorted set commands # x Expired events (events generated every time a key expires) # e Evicted events (events generated when a key is evicted for maxmemory) -# A Alias for g$lshzxe, so that the "AKE" string means all the events. +# t Stream commands +# m Key-miss events (Note: It is not included in the 'A' class) +# A Alias for g$lshzxet, so that the "AKE" string means all the events +# (Except key-miss events which are excluded from 'A' due to their +# unique nature). # # The "notify-keyspace-events" takes as argument a string that is composed # of zero or multiple characters. The empty string means that notifications diff --git a/src/module.c b/src/module.c index b39383637..ed79ee226 100644 --- a/src/module.c +++ b/src/module.c @@ -4807,7 +4807,8 @@ void moduleReleaseGIL(void) { * - REDISMODULE_NOTIFY_EXPIRED: Expiration events * - REDISMODULE_NOTIFY_EVICTED: Eviction events * - REDISMODULE_NOTIFY_STREAM: Stream events - * - REDISMODULE_NOTIFY_ALL: All events + * - REDISMODULE_NOTIFY_KEYMISS: Key-miss events + * - REDISMODULE_NOTIFY_ALL: All events (Excluding REDISMODULE_NOTIFY_KEYMISS) * * We do not distinguish between key events and keyspace events, and it is up * to the module to filter the actions taken based on the key. diff --git a/src/notify.c b/src/notify.c index d6c3ad403..bb1055724 100644 --- a/src/notify.c +++ b/src/notify.c @@ -82,10 +82,10 @@ sds keyspaceEventsFlagsToString(int flags) { if (flags & NOTIFY_EXPIRED) res = sdscatlen(res,"x",1); if (flags & NOTIFY_EVICTED) res = sdscatlen(res,"e",1); if (flags & NOTIFY_STREAM) res = sdscatlen(res,"t",1); - if (flags & NOTIFY_KEY_MISS) res = sdscatlen(res,"m",1); } if (flags & NOTIFY_KEYSPACE) res = sdscatlen(res,"K",1); if (flags & NOTIFY_KEYEVENT) res = sdscatlen(res,"E",1); + if (flags & NOTIFY_KEY_MISS) res = sdscatlen(res,"m",1); return res; } diff --git a/src/redismodule.h b/src/redismodule.h index 637078f2b..e74611f13 100644 --- a/src/redismodule.h +++ b/src/redismodule.h @@ -127,8 +127,8 @@ #define REDISMODULE_NOTIFY_EXPIRED (1<<8) /* x */ #define REDISMODULE_NOTIFY_EVICTED (1<<9) /* e */ #define REDISMODULE_NOTIFY_STREAM (1<<10) /* t */ -#define REDISMODULE_NOTIFY_KEY_MISS (1<<11) /* m */ -#define REDISMODULE_NOTIFY_ALL (REDISMODULE_NOTIFY_GENERIC | REDISMODULE_NOTIFY_STRING | REDISMODULE_NOTIFY_LIST | REDISMODULE_NOTIFY_SET | REDISMODULE_NOTIFY_HASH | REDISMODULE_NOTIFY_ZSET | REDISMODULE_NOTIFY_EXPIRED | REDISMODULE_NOTIFY_EVICTED | REDISMODULE_NOTIFY_STREAM | REDISMODULE_NOTIFY_KEY_MISS) /* A */ +#define REDISMODULE_NOTIFY_KEY_MISS (1<<11) /* m (Note: This one is excluded from REDISMODULE_NOTIFY_ALL on purpose) */ +#define REDISMODULE_NOTIFY_ALL (REDISMODULE_NOTIFY_GENERIC | REDISMODULE_NOTIFY_STRING | REDISMODULE_NOTIFY_LIST | REDISMODULE_NOTIFY_SET | REDISMODULE_NOTIFY_HASH | REDISMODULE_NOTIFY_ZSET | REDISMODULE_NOTIFY_EXPIRED | REDISMODULE_NOTIFY_EVICTED | REDISMODULE_NOTIFY_STREAM) /* A */ /* A special pointer that we can use between the core and the module to signal diff --git a/src/server.h b/src/server.h index 6a081c338..cf423a664 100644 --- a/src/server.h +++ b/src/server.h @@ -413,8 +413,8 @@ typedef long long ustime_t; /* microsecond time type. */ #define NOTIFY_EXPIRED (1<<8) /* x */ #define NOTIFY_EVICTED (1<<9) /* e */ #define NOTIFY_STREAM (1<<10) /* t */ -#define NOTIFY_KEY_MISS (1<<11) /* m */ -#define NOTIFY_ALL (NOTIFY_GENERIC | NOTIFY_STRING | NOTIFY_LIST | NOTIFY_SET | NOTIFY_HASH | NOTIFY_ZSET | NOTIFY_EXPIRED | NOTIFY_EVICTED | NOTIFY_STREAM | NOTIFY_KEY_MISS) /* A flag */ +#define NOTIFY_KEY_MISS (1<<11) /* m (Note: This one is excluded from NOTIFY_ALL on purpose) */ +#define NOTIFY_ALL (NOTIFY_GENERIC | NOTIFY_STRING | NOTIFY_LIST | NOTIFY_SET | NOTIFY_HASH | NOTIFY_ZSET | NOTIFY_EXPIRED | NOTIFY_EVICTED | NOTIFY_STREAM) /* A flag */ /* Get the first bind addr or NULL */ #define NET_FIRST_BIND_ADDR (server.bindaddr_count ? server.bindaddr[0] : NULL) From 5e042dbc05f8c92ff8f4210a39f2be43ae4f0e6e Mon Sep 17 00:00:00 2001 From: lifubang Date: Tue, 4 Feb 2020 17:32:30 +0800 Subject: [PATCH 052/225] fix ssl flag check for redis-cli Signed-off-by: lifubang --- src/redis-cli.c | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/src/redis-cli.c b/src/redis-cli.c index 1919829e1..c1bda0a00 100644 --- a/src/redis-cli.c +++ b/src/redis-cli.c @@ -1595,15 +1595,15 @@ static int parseOptions(int argc, char **argv) { #ifdef USE_OPENSSL } else if (!strcmp(argv[i],"--tls")) { config.tls = 1; - } else if (!strcmp(argv[i],"--sni")) { + } else if (!strcmp(argv[i],"--sni") && !lastarg) { config.sni = argv[++i]; - } else if (!strcmp(argv[i],"--cacertdir")) { + } else if (!strcmp(argv[i],"--cacertdir") && !lastarg) { config.cacertdir = argv[++i]; - } else if (!strcmp(argv[i],"--cacert")) { + } else if (!strcmp(argv[i],"--cacert") && !lastarg) { config.cacert = argv[++i]; - } else if (!strcmp(argv[i],"--cert")) { + } else if (!strcmp(argv[i],"--cert") && !lastarg) { config.cert = argv[++i]; - } else if (!strcmp(argv[i],"--key")) { + } else if (!strcmp(argv[i],"--key") && !lastarg) { config.key = argv[++i]; #endif } else if (!strcmp(argv[i],"-v") || !strcmp(argv[i], "--version")) { @@ -1701,12 +1701,13 @@ static void usage(void) { " -c Enable cluster mode (follow -ASK and -MOVED redirections).\n" #ifdef USE_OPENSSL " --tls Establish a secure TLS connection.\n" -" --cacert CA Certificate file to verify with.\n" -" --cacertdir Directory where trusted CA certificates are stored.\n" +" --sni Server name indication for TLS.\n" +" --cacert CA Certificate file to verify with.\n" +" --cacertdir Directory where trusted CA certificates are stored.\n" " If neither cacert nor cacertdir are specified, the default\n" " system-wide trusted root certs configuration will apply.\n" -" --cert Client certificate to authenticate with.\n" -" --key Private key file to authenticate with.\n" +" --cert Client certificate to authenticate with.\n" +" --key Private key file to authenticate with.\n" #endif " --raw Use raw formatting for replies (default when STDOUT is\n" " not a tty).\n" From dd34f703687ff3ce416fea604b44df1f733dc9ac Mon Sep 17 00:00:00 2001 From: Guy Benoish Date: Mon, 3 Feb 2020 17:19:00 +0530 Subject: [PATCH 053/225] Diskless-load emptyDb-related fixes 1. Call emptyDb even in case of diskless-load: We want modules to get the same FLUSHDB event as disk-based replication. 2. Do not fire any module events when flushing the backups array. 3. Delete redundant call to signalFlushedDb (Called from emptyDb). --- src/db.c | 57 ++++++++++++++++++++++++++++------------------- src/replication.c | 14 +++++++----- src/server.h | 1 + 3 files changed, 44 insertions(+), 28 deletions(-) diff --git a/src/db.c b/src/db.c index 88342ac4d..c80524e94 100644 --- a/src/db.c +++ b/src/db.c @@ -347,7 +347,10 @@ robj *dbUnshareStringValue(redisDb *db, robj *key, robj *o) { * DB number if we want to flush only a single Redis database number. * * Flags are be EMPTYDB_NO_FLAGS if no special flags are specified or - * EMPTYDB_ASYNC if we want the memory to be freed in a different thread + * 1. EMPTYDB_ASYNC if we want the memory to be freed in a different thread. + * 2. EMPTYDB_BACKUP if we want to empty the backup dictionaries created by + * disklessLoadMakeBackups. In that case we only free memory and avoid + * firing module events. * and the function to return ASAP. * * On success the fuction returns the number of keys removed from the @@ -355,6 +358,8 @@ robj *dbUnshareStringValue(redisDb *db, robj *key, robj *o) { * DB number is out of range, and errno is set to EINVAL. */ long long emptyDbGeneric(redisDb *dbarray, int dbnum, int flags, void(callback)(void*)) { int async = (flags & EMPTYDB_ASYNC); + int backup = (flags & EMPTYDB_BACKUP); /* Just free the memory, nothing else */ + RedisModuleFlushInfoV1 fi = {REDISMODULE_FLUSHINFO_VERSION,!async,dbnum}; long long removed = 0; if (dbnum < -1 || dbnum >= server.dbnum) { @@ -362,16 +367,18 @@ long long emptyDbGeneric(redisDb *dbarray, int dbnum, int flags, void(callback)( return -1; } - /* Fire the flushdb modules event. */ - RedisModuleFlushInfoV1 fi = {REDISMODULE_FLUSHINFO_VERSION,!async,dbnum}; - moduleFireServerEvent(REDISMODULE_EVENT_FLUSHDB, - REDISMODULE_SUBEVENT_FLUSHDB_START, - &fi); + /* Pre-flush actions */ + if (!backup) { + /* Fire the flushdb modules event. */ + moduleFireServerEvent(REDISMODULE_EVENT_FLUSHDB, + REDISMODULE_SUBEVENT_FLUSHDB_START, + &fi); - /* Make sure the WATCHed keys are affected by the FLUSH* commands. - * Note that we need to call the function while the keys are still - * there. */ - signalFlushedDb(dbnum); + /* Make sure the WATCHed keys are affected by the FLUSH* commands. + * Note that we need to call the function while the keys are still + * there. */ + signalFlushedDb(dbnum); + } int startdb, enddb; if (dbnum == -1) { @@ -390,20 +397,24 @@ long long emptyDbGeneric(redisDb *dbarray, int dbnum, int flags, void(callback)( dictEmpty(dbarray[j].expires,callback); } } - if (server.cluster_enabled) { - if (async) { - slotToKeyFlushAsync(); - } else { - slotToKeyFlush(); - } - } - if (dbnum == -1) flushSlaveKeysWithExpireList(); - /* Also fire the end event. Note that this event will fire almost - * immediately after the start event if the flush is asynchronous. */ - moduleFireServerEvent(REDISMODULE_EVENT_FLUSHDB, - REDISMODULE_SUBEVENT_FLUSHDB_END, - &fi); + /* Post-flush actions */ + if (!backup) { + if (server.cluster_enabled) { + if (async) { + slotToKeyFlushAsync(); + } else { + slotToKeyFlush(); + } + } + if (dbnum == -1) flushSlaveKeysWithExpireList(); + + /* Also fire the end event. Note that this event will fire almost + * immediately after the start event if the flush is asynchronous. */ + moduleFireServerEvent(REDISMODULE_EVENT_FLUSHDB, + REDISMODULE_SUBEVENT_FLUSHDB_END, + &fi); + } return removed; } diff --git a/src/replication.c b/src/replication.c index a006d0ad1..4843f97d5 100644 --- a/src/replication.c +++ b/src/replication.c @@ -1339,8 +1339,8 @@ void disklessLoadRestoreBackups(redisDb *backup, int restore, int empty_db_flags server.db[i] = backup[i]; } } else { - /* Delete. */ - emptyDbGeneric(backup,-1,empty_db_flags,replicationEmptyDbCallback); + /* Delete (Pass EMPTYDB_BACKUP in order to avoid firing module events) . */ + emptyDbGeneric(backup,-1,empty_db_flags|EMPTYDB_BACKUP,replicationEmptyDbCallback); for (int i=0; i Date: Mon, 10 Feb 2020 16:32:46 +0900 Subject: [PATCH 054/225] [FIX] revisit CVE-2015-8080 vulnerability --- deps/lua/src/lua_struct.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/deps/lua/src/lua_struct.c b/deps/lua/src/lua_struct.c index 4d5f027b8..c58c8e72b 100644 --- a/deps/lua/src/lua_struct.c +++ b/deps/lua/src/lua_struct.c @@ -89,12 +89,14 @@ typedef struct Header { } Header; -static int getnum (const char **fmt, int df) { +static int getnum (lua_State *L, const char **fmt, int df) { if (!isdigit(**fmt)) /* no number? */ return df; /* return default value */ else { int a = 0; do { + if (a > (INT_MAX / 10) || a * 10 > (INT_MAX - (**fmt - '0'))) + luaL_error(L, "integral size overflow"); a = a*10 + *((*fmt)++) - '0'; } while (isdigit(**fmt)); return a; @@ -115,9 +117,9 @@ static size_t optsize (lua_State *L, char opt, const char **fmt) { case 'f': return sizeof(float); case 'd': return sizeof(double); case 'x': return 1; - case 'c': return getnum(fmt, 1); + case 'c': return getnum(L, fmt, 1); case 'i': case 'I': { - int sz = getnum(fmt, sizeof(int)); + int sz = getnum(L, fmt, sizeof(int)); if (sz > MAXINTSIZE) luaL_error(L, "integral size %d is larger than limit of %d", sz, MAXINTSIZE); @@ -150,7 +152,7 @@ static void controloptions (lua_State *L, int opt, const char **fmt, case '>': h->endian = BIG; return; case '<': h->endian = LITTLE; return; case '!': { - int a = getnum(fmt, MAXALIGN); + int a = getnum(L, fmt, MAXALIGN); if (!isp2(a)) luaL_error(L, "alignment %d is not a power of 2", a); h->align = a; From 54f5499aee18f9cc3901b0bd4699f29d49bb9dce Mon Sep 17 00:00:00 2001 From: lifubang Date: Wed, 12 Feb 2020 16:34:22 +0800 Subject: [PATCH 055/225] correct help info for --user and --pass Signed-off-by: lifubang --- src/redis-cli.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/redis-cli.c b/src/redis-cli.c index c1bda0a00..1d79e0db0 100644 --- a/src/redis-cli.c +++ b/src/redis-cli.c @@ -1688,8 +1688,8 @@ static void usage(void) { " You can also use the " REDIS_CLI_AUTH_ENV " environment\n" " variable to pass this password more safely\n" " (if both are used, this argument takes predecence).\n" -" -user Used to send ACL style 'AUTH username pass'. Needs -a.\n" -" -pass Alias of -a for consistency with the new --user option.\n" +" --user Used to send ACL style 'AUTH username pass'. Needs -a.\n" +" --pass Alias of -a for consistency with the new --user option.\n" " -u Server URI.\n" " -r Execute specified command N times.\n" " -i When -r is used, waits seconds per command.\n" From 5e762d847cc046e830b899b4d885d971d11df42b Mon Sep 17 00:00:00 2001 From: Khem Raj Date: Sat, 21 Dec 2019 11:17:50 -0800 Subject: [PATCH 056/225] Mark extern definition of SDS_NOINIT in sds.h This helps in avoiding multiple definition of this variable, its also defined globally in sds.c Signed-off-by: Khem Raj --- src/sds.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sds.h b/src/sds.h index 1bdb60dec..adcc12c0a 100644 --- a/src/sds.h +++ b/src/sds.h @@ -34,7 +34,7 @@ #define __SDS_H #define SDS_MAX_PREALLOC (1024*1024) -const char *SDS_NOINIT; +extern const char *SDS_NOINIT; #include #include From a788c373e61b9ad17e09530da1a817664a1f84fa Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 7 Feb 2020 18:12:45 +0100 Subject: [PATCH 057/225] Tracking: minor change of names and new INFO field. --- src/config.c | 2 +- src/server.c | 2 ++ src/server.h | 3 ++- src/tracking.c | 8 ++++++-- 4 files changed, 11 insertions(+), 4 deletions(-) diff --git a/src/config.c b/src/config.c index a6b374817..b2e5fc12e 100644 --- a/src/config.c +++ b/src/config.c @@ -2160,7 +2160,7 @@ standardConfig configs[] = { createIntConfig("list-compress-depth", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.list_compress_depth, 0, INTEGER_CONFIG, NULL, NULL), createIntConfig("rdb-key-save-delay", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.rdb_key_save_delay, 0, INTEGER_CONFIG, NULL, NULL), createIntConfig("key-load-delay", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.key_load_delay, 0, INTEGER_CONFIG, NULL, NULL), - createIntConfig("tracking-table-max-fill", NULL, MODIFIABLE_CONFIG, 0, 100, server.tracking_table_max_fill, 10, INTEGER_CONFIG, NULL, NULL), /* Default: 10% tracking table max fill. */ + createIntConfig("tracking-table-max-fill", NULL, MODIFIABLE_CONFIG, 0, 100, server.tracking_table_max_keys, 1000000, INTEGER_CONFIG, NULL, NULL), /* Default: 10% tracking table max number of keys tracked. */ createIntConfig("active-expire-effort", NULL, MODIFIABLE_CONFIG, 1, 10, server.active_expire_effort, 1, INTEGER_CONFIG, NULL, NULL), /* From 1 to 10. */ createIntConfig("hz", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.config_hz, CONFIG_DEFAULT_HZ, INTEGER_CONFIG, NULL, updateHZ), createIntConfig("min-replicas-to-write", "min-slaves-to-write", MODIFIABLE_CONFIG, 0, INT_MAX, server.repl_min_slaves_to_write, 0, INTEGER_CONFIG, NULL, updateGoodSlaves), diff --git a/src/server.c b/src/server.c index 2cf8dbf19..910cf5410 100644 --- a/src/server.c +++ b/src/server.c @@ -4221,6 +4221,7 @@ sds genRedisInfoString(const char *section) { "active_defrag_misses:%lld\r\n" "active_defrag_key_hits:%lld\r\n" "active_defrag_key_misses:%lld\r\n" + "tracking_total_keys:%lld\r\n" "tracking_total_items:%lld\r\n", server.stat_numconnections, server.stat_numcommands, @@ -4249,6 +4250,7 @@ sds genRedisInfoString(const char *section) { server.stat_active_defrag_misses, server.stat_active_defrag_key_hits, server.stat_active_defrag_key_misses, + (unsigned long long) trackingGetTotalKeys(), (unsigned long long) trackingGetTotalItems()); } diff --git a/src/server.h b/src/server.h index a30db6b57..3e055a7db 100644 --- a/src/server.h +++ b/src/server.h @@ -1306,7 +1306,7 @@ struct redisServer { list *ready_keys; /* List of readyList structures for BLPOP & co */ /* Client side caching. */ unsigned int tracking_clients; /* # of clients with tracking enabled.*/ - int tracking_table_max_fill; /* Max fill percentage. */ + int tracking_table_max_keys; /* Max number of keys in tracking table. */ /* Sort parameters - qsort_r() is only available under BSD so we * have to take this state global, in order to pass it to sortCompare() */ int sort_desc; @@ -1655,6 +1655,7 @@ void trackingInvalidateKey(robj *keyobj); void trackingInvalidateKeysOnFlush(int dbid); void trackingLimitUsedSlots(void); uint64_t trackingGetTotalItems(void); +uint64_t trackingGetTotalKeys(void); /* List data type */ void listTypeTryConversion(robj *subject, robj *value); diff --git a/src/tracking.c b/src/tracking.c index ecb7fbdcc..9c1c9620c 100644 --- a/src/tracking.c +++ b/src/tracking.c @@ -223,8 +223,8 @@ void trackingInvalidateKeysOnFlush(int dbid) { void trackingLimitUsedSlots(void) { static unsigned int timeout_counter = 0; if (TrackingTable == NULL) return; - if (server.tracking_table_max_fill == 0) return; /* No limits set. */ - size_t max_keys = server.tracking_table_max_fill; + if (server.tracking_table_max_keys == 0) return; /* No limits set. */ + size_t max_keys = server.tracking_table_max_keys; if (raxSize(TrackingTable) <= max_keys) { timeout_counter = 0; return; /* Limit not reached. */ @@ -264,3 +264,7 @@ void trackingLimitUsedSlots(void) { uint64_t trackingGetTotalItems(void) { return TrackingTableTotalItems; } + +uint64_t trackingGetTotalKeys(void) { + return raxSize(TrackingTable); +} From 3e8c69a9de2a8ff56b60652cda8566cfa22f3004 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 10 Feb 2020 13:42:18 +0100 Subject: [PATCH 058/225] Tracking: always reply with an array of keys. --- src/pubsub.c | 8 ++++++-- src/tracking.c | 5 ++++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/pubsub.c b/src/pubsub.c index 994dd9734..5cb4298e0 100644 --- a/src/pubsub.c +++ b/src/pubsub.c @@ -35,7 +35,11 @@ int clientSubscriptionsCount(client *c); * Pubsub client replies API *----------------------------------------------------------------------------*/ -/* Send a pubsub message of type "message" to the client. */ +/* Send a pubsub message of type "message" to the client. + * Normally 'msg' is a Redis object containing the string to send as + * message. However if the caller sets 'msg' as NULL, it will be able + * to send a special message (for instance an Array type) by using the + * addReply*() API family. */ void addReplyPubsubMessage(client *c, robj *channel, robj *msg) { if (c->resp == 2) addReply(c,shared.mbulkhdr[3]); @@ -43,7 +47,7 @@ void addReplyPubsubMessage(client *c, robj *channel, robj *msg) { addReplyPushLen(c,3); addReply(c,shared.messagebulk); addReplyBulk(c,channel); - addReplyBulk(c,msg); + if (msg) addReplyBulk(c,msg); } /* Send a pubsub message of type "pmessage" to the client. The difference diff --git a/src/tracking.c b/src/tracking.c index 9c1c9620c..3122563ac 100644 --- a/src/tracking.c +++ b/src/tracking.c @@ -132,13 +132,16 @@ void sendTrackingMessage(client *c, char *keyname, size_t keylen) { if (c->resp > 2) { addReplyPushLen(c,2); addReplyBulkCBuffer(c,"invalidate",10); + addReplyArrayLen(c,1); addReplyBulkCBuffer(c,keyname,keylen); } else if (using_redirection && c->flags & CLIENT_PUBSUB) { /* We use a static object to speedup things, however we assume * that addReplyPubsubMessage() will not take a reference. */ robj keyobj; initStaticStringObject(keyobj,keyname); - addReplyPubsubMessage(c,TrackingChannelName,&keyobj); + addReplyPubsubMessage(c,TrackingChannelName,NULL); + addReplyArrayLen(c,1); + addReplyBulk(c,&keyobj); serverAssert(keyobj.refcount == 1); } } From 77da960815a49e6a0a5cd0384c397fb1a88b52ee Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 10 Feb 2020 17:18:11 +0100 Subject: [PATCH 059/225] Tracking: BCAST: parsing of the options + skeleton. --- src/networking.c | 63 +++++++++++++++++++++++++++++++++++++----------- src/server.c | 5 +++- src/server.h | 8 +++++- src/tracking.c | 16 +++++++++--- 4 files changed, 73 insertions(+), 19 deletions(-) diff --git a/src/networking.c b/src/networking.c index 2b0f7464a..690a134f1 100644 --- a/src/networking.c +++ b/src/networking.c @@ -154,6 +154,7 @@ client *createClient(connection *conn) { c->peerid = NULL; c->client_list_node = NULL; c->client_tracking_redirection = 0; + c->client_tracking_prefix_nodes = NULL; c->auth_callback = NULL; c->auth_callback_privdata = NULL; c->auth_module = NULL; @@ -2219,38 +2220,72 @@ NULL UNIT_MILLISECONDS) != C_OK) return; pauseClients(duration); addReply(c,shared.ok); - } else if (!strcasecmp(c->argv[1]->ptr,"tracking") && - (c->argc == 3 || c->argc == 5)) - { - /* CLIENT TRACKING (on|off) [REDIRECT ] */ + } else if (!strcasecmp(c->argv[1]->ptr,"tracking") && c->argc >= 3) { + /* CLIENT TRACKING (on|off) [REDIRECT ] [BCAST] [PREFIX first] + * [PREFIX second] ... */ long long redir = 0; + int bcast = 0; + robj **prefix; + size_t numprefix = 0; - /* Parse the redirection option: we'll require the client with - * the specified ID to exist right now, even if it is possible - * it will get disconnected later. */ - if (c->argc == 5) { - if (strcasecmp(c->argv[3]->ptr,"redirect") != 0) { - addReply(c,shared.syntaxerr); - return; - } else { - if (getLongLongFromObjectOrReply(c,c->argv[4],&redir,NULL) != + /* Parse the options. */ + if (for int j = 3; j < argc; j++) { + int moreargs = (c->argc-1) - j; + + if (!strcasecmp(c->argv[j]->ptr,"redirect") && moreargs) { + j++; + if (getLongLongFromObjectOrReply(c,c->argv[j],&redir,NULL) != C_OK) return; + /* We will require the client with the specified ID to exist + * right now, even if it is possible that it gets disconnected + * later. Still a valid sanity check. */ if (lookupClientByID(redir) == NULL) { addReplyError(c,"The client ID you want redirect to " "does not exist"); return; } + } else if (!strcasecmp(c->argv[j]->ptr,"bcast")) { + bcast++; + } else if (!strcasecmp(c->argv[j]->ptr,"prefix") && morearg) { + j++; + prefix = zrealloc(sizeof(robj*)*(numprefix+1)); + prefix[numprefix++] = argv[j]; + } else { + addReply(c,shared.syntaxerr); + return; } } + /* Make sure options are compatible among each other and with the + * current state of the client. */ + if (!bcast && numprefix) { + addReplyError("PREFIX option requires BCAST mode to be enabled"); + zfree(prefix); + return; + } + + if (client->flags & CLIENT_TRACKING) { + int oldbcast = !!client->flags & CLIENT_TRACKING_BCAST; + if (oldbcast != bcast) { + } + addReplyError( + "You can't switch BCAST mode on/off before disabling " + "tracking for this client, and then re-enabling it with " + "a different mode."); + zfree(prefix); + return; + } + + /* Options are ok: enable or disable the tracking for this client. */ if (!strcasecmp(c->argv[2]->ptr,"on")) { - enableTracking(c,redir); + enableTracking(c,redir,bcast,prefix,numprefix); } else if (!strcasecmp(c->argv[2]->ptr,"off")) { disableTracking(c); } else { addReply(c,shared.syntaxerr); return; } + zfree(prefix); addReply(c,shared.ok); } else if (!strcasecmp(c->argv[1]->ptr,"getredir") && c->argc == 2) { /* CLIENT GETREDIR */ diff --git a/src/server.c b/src/server.c index 910cf5410..1001fa4f7 100644 --- a/src/server.c +++ b/src/server.c @@ -3310,8 +3310,11 @@ void call(client *c, int flags) { if (c->cmd->flags & CMD_READONLY) { client *caller = (c->flags & CLIENT_LUA && server.lua_caller) ? server.lua_caller : c; - if (caller->flags & CLIENT_TRACKING) + if (caller->flags & CLIENT_TRACKING && + !(caller->flags & CLIENT_TRACKING_BCAST)) + { trackingRememberKeys(caller); + } } server.fixed_time_expire--; diff --git a/src/server.h b/src/server.h index 3e055a7db..d3ca0d01b 100644 --- a/src/server.h +++ b/src/server.h @@ -247,6 +247,7 @@ typedef long long ustime_t; /* microsecond time type. */ #define CLIENT_TRACKING (1ULL<<31) /* Client enabled keys tracking in order to perform client side caching. */ #define CLIENT_TRACKING_BROKEN_REDIR (1ULL<<32) /* Target client is invalid. */ +#define CLIENT_TRACKING_BCAST (1ULL<<33) /* Tracking in BCAST mode. */ /* Client block type (btype field in client structure) * if CLIENT_BLOCKED flag is set. */ @@ -822,6 +823,11 @@ typedef struct client { * invalidation messages for keys fetched by this client will be send to * the specified client ID. */ uint64_t client_tracking_redirection; + list *client_tracking_prefix_nodes; /* This list contains listNode pointers + to the nodes we have in every list + of clients in the tracking bcast + table. This way we can remove our + client in O(1) for each list. */ /* Response buffer */ int bufpos; @@ -1648,7 +1654,7 @@ void addReplyStatusFormat(client *c, const char *fmt, ...); #endif /* Client side caching (tracking mode) */ -void enableTracking(client *c, uint64_t redirect_to); +void enableTracking(client *c, uint64_t redirect_to, int bcast, robj **prefix, size_t numprefix); void disableTracking(client *c); void trackingRememberKeys(client *c); void trackingInvalidateKey(robj *keyobj); diff --git a/src/tracking.c b/src/tracking.c index 3122563ac..413b21328 100644 --- a/src/tracking.c +++ b/src/tracking.c @@ -42,6 +42,7 @@ * Clients will normally take frequently requested objects in memory, removing * them when invalidation messages are received. */ rax *TrackingTable = NULL; +rax *PrefixTable = NULL; uint64_t TrackingTableTotalItems = 0; /* Total number of IDs stored across the whole tracking table. This givesn an hint about the total memory we @@ -68,16 +69,25 @@ void disableTracking(client *c) { * eventually get freed, we'll send a message to the original client to * inform it of the condition. Multiple clients can redirect the invalidation * messages to the same client ID. */ -void enableTracking(client *c, uint64_t redirect_to) { - if (c->flags & CLIENT_TRACKING) return; +void enableTracking(client *c, uint64_t redirect_to, int bcast, robj **prefix, size_t numprefix) { c->flags |= CLIENT_TRACKING; c->flags &= ~CLIENT_TRACKING_BROKEN_REDIR; c->client_tracking_redirection = redirect_to; - server.tracking_clients++; + if (!(c->flags & CLIENT_TRACKING)) server.tracking_clients++; if (TrackingTable == NULL) { TrackingTable = raxNew(); + PrefixTable = raxNew(); TrackingChannelName = createStringObject("__redis__:invalidate",20); } + + if (bcast) { + c->flags |= CLIENT_TRACKING_BCAST; + if (numprefix == 0) enableBcastTrackingForPrefix(c,"",0); + for (int j = 0; j < numprefix; j++) { + sds sdsprefix = prefix[j]->ptr; + enableBcastTrackingForPrefix(c,sdsprefix,sdslen(prefix)); + } + } } /* This function is called after the excution of a readonly command in the From abb81c635143dc329eefb5729acec0a62d53cab1 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 11 Feb 2020 17:26:27 +0100 Subject: [PATCH 060/225] Tracking: BCAST: registration in the prefix table. --- src/networking.c | 21 +++++++++--------- src/server.h | 9 +++----- src/tracking.c | 57 +++++++++++++++++++++++++++++++++++++++++++++--- 3 files changed, 67 insertions(+), 20 deletions(-) diff --git a/src/networking.c b/src/networking.c index 690a134f1..344b76260 100644 --- a/src/networking.c +++ b/src/networking.c @@ -154,7 +154,7 @@ client *createClient(connection *conn) { c->peerid = NULL; c->client_list_node = NULL; c->client_tracking_redirection = 0; - c->client_tracking_prefix_nodes = NULL; + c->client_tracking_prefixes = NULL; c->auth_callback = NULL; c->auth_callback_privdata = NULL; c->auth_module = NULL; @@ -2028,7 +2028,6 @@ int clientSetNameOrReply(client *c, robj *name) { void clientCommand(client *c) { listNode *ln; listIter li; - client *client; if (c->argc == 2 && !strcasecmp(c->argv[1]->ptr,"help")) { const char *help[] = { @@ -2142,7 +2141,7 @@ NULL /* Iterate clients killing all the matching clients. */ listRewind(server.clients,&li); while ((ln = listNext(&li)) != NULL) { - client = listNodeValue(ln); + client *client = listNodeValue(ln); if (addr && strcmp(getClientPeerId(client),addr) != 0) continue; if (type != -1 && getClientType(client) != type) continue; if (id != 0 && client->id != id) continue; @@ -2229,7 +2228,7 @@ NULL size_t numprefix = 0; /* Parse the options. */ - if (for int j = 3; j < argc; j++) { + for (int j = 3; j < c->argc; j++) { int moreargs = (c->argc-1) - j; if (!strcasecmp(c->argv[j]->ptr,"redirect") && moreargs) { @@ -2246,10 +2245,10 @@ NULL } } else if (!strcasecmp(c->argv[j]->ptr,"bcast")) { bcast++; - } else if (!strcasecmp(c->argv[j]->ptr,"prefix") && morearg) { + } else if (!strcasecmp(c->argv[j]->ptr,"prefix") && moreargs) { j++; - prefix = zrealloc(sizeof(robj*)*(numprefix+1)); - prefix[numprefix++] = argv[j]; + prefix = zrealloc(prefix,sizeof(robj*)*(numprefix+1)); + prefix[numprefix++] = c->argv[j]; } else { addReply(c,shared.syntaxerr); return; @@ -2259,16 +2258,16 @@ NULL /* Make sure options are compatible among each other and with the * current state of the client. */ if (!bcast && numprefix) { - addReplyError("PREFIX option requires BCAST mode to be enabled"); + addReplyError(c,"PREFIX option requires BCAST mode to be enabled"); zfree(prefix); return; } - if (client->flags & CLIENT_TRACKING) { - int oldbcast = !!client->flags & CLIENT_TRACKING_BCAST; + if (c->flags & CLIENT_TRACKING) { + int oldbcast = !!c->flags & CLIENT_TRACKING_BCAST; if (oldbcast != bcast) { } - addReplyError( + addReplyError(c, "You can't switch BCAST mode on/off before disabling " "tracking for this client, and then re-enabling it with " "a different mode."); diff --git a/src/server.h b/src/server.h index d3ca0d01b..725c3cbc8 100644 --- a/src/server.h +++ b/src/server.h @@ -823,12 +823,9 @@ typedef struct client { * invalidation messages for keys fetched by this client will be send to * the specified client ID. */ uint64_t client_tracking_redirection; - list *client_tracking_prefix_nodes; /* This list contains listNode pointers - to the nodes we have in every list - of clients in the tracking bcast - table. This way we can remove our - client in O(1) for each list. */ - + rax *client_tracking_prefixes; /* A dictionary of prefixes we are already + subscribed to in BCAST mode, in the + context of client side caching. */ /* Response buffer */ int bufpos; char buf[PROTO_REPLY_CHUNK_BYTES]; diff --git a/src/tracking.c b/src/tracking.c index 413b21328..9f46275a4 100644 --- a/src/tracking.c +++ b/src/tracking.c @@ -49,6 +49,15 @@ uint64_t TrackingTableTotalItems = 0; /* Total number of IDs stored across are using server side for CSC. */ robj *TrackingChannelName; +/* This is the structure that we have as value of the PrefixTable, and + * represents the list of keys modified, and the list of clients that need + * to be notified, for a given prefix. */ +typedef struct bcastState { + rax *keys; /* Keys modified in the current event loop cycle. */ + rax *clients; /* Clients subscribed to the notification events for this + prefix. */ +} bcastState; + /* Remove the tracking state from the client 'c'. Note that there is not much * to do for us here, if not to decrement the counter of the clients in * tracking mode, because we just store the ID of the client in the tracking @@ -56,9 +65,51 @@ robj *TrackingChannelName; * client with many entries in the table is removed, it would cost a lot of * time to do the cleanup. */ void disableTracking(client *c) { + /* If this client is in broadcasting mode, we need to unsubscribe it + * from all the prefixes it is registered to. */ + if (c->flags & CLIENT_TRACKING_BCAST) { + raxIterator ri; + raxStart(&ri,c->client_tracking_prefixes); + raxSeek(&ri,"^",NULL,0); + while(raxNext(&ri)) { + bcastState *bs = raxFind(PrefixTable,ri.key,ri.key_len); + serverAssert(bs != raxNotFound); + raxRemove(bs->clients,(unsigned char*)&c,sizeof(c),NULL); + /* Was it the last client? Remove the prefix from the + * table. */ + if (raxSize(bs->clients) == 0) { + raxFree(bs->clients); + raxFree(bs->keys); + zfree(bs); + raxRemove(PrefixTable,ri.key,ri.key_len,NULL); + } + } + raxStop(&ri); + } + + /* Clear flags and adjust the count. */ if (c->flags & CLIENT_TRACKING) { server.tracking_clients--; - c->flags &= ~(CLIENT_TRACKING|CLIENT_TRACKING_BROKEN_REDIR); + c->flags &= ~(CLIENT_TRACKING|CLIENT_TRACKING_BROKEN_REDIR| + CLIENT_TRACKING_BCAST); + } +} + +/* Set the client 'c' to track the prefix 'prefix'. If the client 'c' is + * already registered for the specified prefix, no operation is performed. */ +void enableBcastTrackingForPrefix(client *c, char *prefix, size_t plen) { + bcastState *bs = raxFind(PrefixTable,(unsigned char*)prefix,sdslen(prefix)); + /* If this is the first client subscribing to such prefix, create + * the prefix in the table. */ + if (bs == raxNotFound) { + bs = zmalloc(sizeof(*bs)); + bs->keys = raxNew(); + bs->clients = raxNew(); + raxInsert(PrefixTable,(unsigned char*)prefix,plen,bs,NULL); + } + if (raxTryInsert(bs->clients,(unsigned char*)&c,sizeof(c),NULL,NULL)) { + raxInsert(c->client_tracking_prefixes, + (unsigned char*)prefix,plen,NULL,NULL); } } @@ -83,9 +134,9 @@ void enableTracking(client *c, uint64_t redirect_to, int bcast, robj **prefix, s if (bcast) { c->flags |= CLIENT_TRACKING_BCAST; if (numprefix == 0) enableBcastTrackingForPrefix(c,"",0); - for (int j = 0; j < numprefix; j++) { + for (size_t j = 0; j < numprefix; j++) { sds sdsprefix = prefix[j]->ptr; - enableBcastTrackingForPrefix(c,sdsprefix,sdslen(prefix)); + enableBcastTrackingForPrefix(c,sdsprefix,sdslen(sdsprefix)); } } } From d4fe79a1740ca41fbc5524640c1a3b0b97e2bf81 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 11 Feb 2020 18:11:59 +0100 Subject: [PATCH 061/225] Tracking: BCAST: broadcasting of keys in prefixes implemented. --- src/server.h | 1 + src/tracking.c | 104 ++++++++++++++++++++++++++++++++++++++++++++----- 2 files changed, 95 insertions(+), 10 deletions(-) diff --git a/src/server.h b/src/server.h index 725c3cbc8..439bbc393 100644 --- a/src/server.h +++ b/src/server.h @@ -1659,6 +1659,7 @@ void trackingInvalidateKeysOnFlush(int dbid); void trackingLimitUsedSlots(void); uint64_t trackingGetTotalItems(void); uint64_t trackingGetTotalKeys(void); +void trackingBroadcastInvalidationMessages(void); /* List data type */ void listTypeTryConversion(robj *subject, robj *value); diff --git a/src/tracking.c b/src/tracking.c index 9f46275a4..345c5f1ad 100644 --- a/src/tracking.c +++ b/src/tracking.c @@ -167,7 +167,17 @@ void trackingRememberKeys(client *c) { getKeysFreeResult(keys); } -void sendTrackingMessage(client *c, char *keyname, size_t keylen) { +/* Given a key name, this function sends an invalidation message in the + * proper channel (depending on RESP version: PubSub or Push message) and + * to the proper client (in case fo redirection), in the context of the + * client 'c' with tracking enabled. + * + * In case the 'proto' argument is non zero, the function will assume that + * 'keyname' points to a buffer of 'keylen' bytes already expressed in the + * form of Redis RESP protocol, representing an array of keys to send + * to the client as value of the invalidation. This is used in BCAST mode + * in order to optimized the implementation to use less CPU time. */ +void sendTrackingMessage(client *c, char *keyname, size_t keylen, int proto) { int using_redirection = 0; if (c->client_tracking_redirection) { client *redir = lookupClientByID(c->client_tracking_redirection); @@ -193,18 +203,38 @@ void sendTrackingMessage(client *c, char *keyname, size_t keylen) { if (c->resp > 2) { addReplyPushLen(c,2); addReplyBulkCBuffer(c,"invalidate",10); - addReplyArrayLen(c,1); - addReplyBulkCBuffer(c,keyname,keylen); } else if (using_redirection && c->flags & CLIENT_PUBSUB) { /* We use a static object to speedup things, however we assume * that addReplyPubsubMessage() will not take a reference. */ - robj keyobj; - initStaticStringObject(keyobj,keyname); addReplyPubsubMessage(c,TrackingChannelName,NULL); - addReplyArrayLen(c,1); - addReplyBulk(c,&keyobj); - serverAssert(keyobj.refcount == 1); } + + /* Send the "value" part, which is the array of keys. */ + if (proto) { + addReplyProto(c,keyname,keylen); + } else { + addReplyArrayLen(c,1); + addReplyBulkCBuffer(c,keyname,keylen); + } +} + +/* This function is called when a key is modified in Redis and in the case + * we have at least one client with the BCAST mode enabled. + * Its goal is to set the key in the right broadcast state if the key + * matches one or more prefixes in the prefix table. Later when we + * return to the event loop, we'll send invalidation messages to the + * clients subscribed to each prefix. */ +void trackingRememberKeyToBroadcast(char *keyname, size_t keylen) { + raxIterator ri; + raxStart(&ri,PrefixTable); + raxSeek(&ri,"^",NULL,0); + while(raxNext(&ri)) { + if (keylen > ri.key_len) continue; + if (memcmp(ri.key,keyname,ri.key_len) != 0) continue; + bcastState *bs = ri.data; + raxTryInsert(bs->keys,(unsigned char*)keyname,keylen,NULL,NULL); + } + raxStop(&ri); } /* This function is called from signalModifiedKey() or other places in Redis @@ -214,6 +244,10 @@ void sendTrackingMessage(client *c, char *keyname, size_t keylen) { void trackingInvalidateKey(robj *keyobj) { if (TrackingTable == NULL) return; sds sdskey = keyobj->ptr; + + if (raxSize(PrefixTable) > 0) + trackingRememberKeyToBroadcast(sdskey,sdslen(sdskey)); + rax *ids = raxFind(TrackingTable,(unsigned char*)sdskey,sdslen(sdskey)); if (ids == raxNotFound) return;; @@ -225,7 +259,7 @@ void trackingInvalidateKey(robj *keyobj) { memcpy(&id,ri.key,sizeof(id)); client *c = lookupClientByID(id); if (c == NULL || !(c->flags & CLIENT_TRACKING)) continue; - sendTrackingMessage(c,sdskey,sdslen(sdskey)); + sendTrackingMessage(c,sdskey,sdslen(sdskey),0); } raxStop(&ri); @@ -262,7 +296,7 @@ void trackingInvalidateKeysOnFlush(int dbid) { while ((ln = listNext(&li)) != NULL) { client *c = listNodeValue(ln); if (c->flags & CLIENT_TRACKING) { - sendTrackingMessage(c,"",1); + sendTrackingMessage(c,"",1,0); } } } @@ -323,6 +357,56 @@ void trackingLimitUsedSlots(void) { timeout_counter++; } +/* This function will run the prefixes of clients in BCAST mode and + * keys that were modified about each prefix, and will send the + * notifications to each client in each prefix. */ +void trackingBroadcastInvalidationMessages(void) { + raxIterator ri, ri2; + raxStart(&ri,PrefixTable); + raxSeek(&ri,"^",NULL,0); + while(raxNext(&ri)) { + bcastState *bs = ri.data; + /* Create the array reply with the list of keys once, then send + * it to all the clients subscribed to this prefix. */ + char buf[32]; + size_t len = ll2string(buf,sizeof(buf),raxSize(bs->keys)); + sds proto = sdsempty(); + proto = sdsMakeRoomFor(proto,raxSize(bs->keys)*15); + proto = sdscatlen(proto,"*",1); + proto = sdscatlen(proto,buf,len); + proto = sdscatlen(proto,"\r\n",2); + raxStart(&ri2,bs->keys); + raxSeek(&ri2,"^",NULL,0); + while(raxNext(&ri2)) { + len = ll2string(buf,sizeof(buf),ri2.key_len); + sds proto = sdsnewlen("$",1); + proto = sdscatlen(proto,ri2.key,ri2.key_len); + proto = sdscatlen(proto,"\r\n",2); + } + raxStop(&ri2); + + /* Send this array of keys to every client in the list. */ + raxStart(&ri2,bs->clients); + raxSeek(&ri2,"^",NULL,0); + while(raxNext(&ri2)) { + client *c; + memcpy(&c,ri2.key,sizeof(c)); + sendTrackingMessage(c,proto,sdslen(proto),1); + } + raxStop(&ri2); + + /* Clean up: we can remove everything from this state, because we + * want to only track the new keys that will be accumulated starting + * from now. */ + sdsfree(proto); + raxFree(bs->clients); + raxFree(bs->keys); + bs->clients = raxNew(); + bs->keys = raxNew(); + } + raxStop(&ri); +} + /* This is just used in order to access the amount of used slots in the * tracking table. */ uint64_t trackingGetTotalItems(void) { From 6fb1aa2381d321203d1cfceb0d0779ae2f469d8a Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 12 Feb 2020 19:22:04 +0100 Subject: [PATCH 062/225] Tracking: BCAST: basic feature now works. --- src/networking.c | 2 +- src/server.c | 4 +++ src/tracking.c | 89 +++++++++++++++++++++++++++--------------------- 3 files changed, 55 insertions(+), 40 deletions(-) diff --git a/src/networking.c b/src/networking.c index 344b76260..46534253e 100644 --- a/src/networking.c +++ b/src/networking.c @@ -2224,7 +2224,7 @@ NULL * [PREFIX second] ... */ long long redir = 0; int bcast = 0; - robj **prefix; + robj **prefix = NULL; size_t numprefix = 0; /* Parse the options. */ diff --git a/src/server.c b/src/server.c index 1001fa4f7..22c81070c 100644 --- a/src/server.c +++ b/src/server.c @@ -2124,6 +2124,10 @@ void beforeSleep(struct aeEventLoop *eventLoop) { if (listLength(server.unblocked_clients)) processUnblockedClients(); + /* Send the invalidation messages to clients participating to the + * client side caching protocol in broadcasting (BCAST) mode. */ + trackingBroadcastInvalidationMessages(); + /* Write the AOF buffer on disk */ flushAppendOnlyFile(0); diff --git a/src/tracking.c b/src/tracking.c index 345c5f1ad..672b886a3 100644 --- a/src/tracking.c +++ b/src/tracking.c @@ -85,6 +85,8 @@ void disableTracking(client *c) { } } raxStop(&ri); + raxFree(c->client_tracking_prefixes); + c->client_tracking_prefixes = NULL; } /* Clear flags and adjust the count. */ @@ -108,6 +110,8 @@ void enableBcastTrackingForPrefix(client *c, char *prefix, size_t plen) { raxInsert(PrefixTable,(unsigned char*)prefix,plen,bs,NULL); } if (raxTryInsert(bs->clients,(unsigned char*)&c,sizeof(c),NULL,NULL)) { + if (c->client_tracking_prefixes == NULL) + c->client_tracking_prefixes = raxNew(); raxInsert(c->client_tracking_prefixes, (unsigned char*)prefix,plen,NULL,NULL); } @@ -121,10 +125,10 @@ void enableBcastTrackingForPrefix(client *c, char *prefix, size_t plen) { * inform it of the condition. Multiple clients can redirect the invalidation * messages to the same client ID. */ void enableTracking(client *c, uint64_t redirect_to, int bcast, robj **prefix, size_t numprefix) { - c->flags |= CLIENT_TRACKING; - c->flags &= ~CLIENT_TRACKING_BROKEN_REDIR; - c->client_tracking_redirection = redirect_to; if (!(c->flags & CLIENT_TRACKING)) server.tracking_clients++; + c->flags |= CLIENT_TRACKING; + c->flags &= ~(CLIENT_TRACKING_BROKEN_REDIR|CLIENT_TRACKING_BCAST); + c->client_tracking_redirection = redirect_to; if (TrackingTable == NULL) { TrackingTable = raxNew(); PrefixTable = raxNew(); @@ -229,10 +233,11 @@ void trackingRememberKeyToBroadcast(char *keyname, size_t keylen) { raxStart(&ri,PrefixTable); raxSeek(&ri,"^",NULL,0); while(raxNext(&ri)) { - if (keylen > ri.key_len) continue; - if (memcmp(ri.key,keyname,ri.key_len) != 0) continue; - bcastState *bs = ri.data; - raxTryInsert(bs->keys,(unsigned char*)keyname,keylen,NULL,NULL); + if (ri.key_len > keylen) continue; + if (ri.key_len != 0 && memcmp(ri.key,keyname,ri.key_len) != 0) + continue; + bcastState *bs = ri.data; + raxTryInsert(bs->keys,(unsigned char*)keyname,keylen,NULL,NULL); } raxStop(&ri); } @@ -362,46 +367,52 @@ void trackingLimitUsedSlots(void) { * notifications to each client in each prefix. */ void trackingBroadcastInvalidationMessages(void) { raxIterator ri, ri2; + + /* Return ASAP if there is nothing to do here. */ + if (TrackingTable == NULL || !server.tracking_clients) return; + raxStart(&ri,PrefixTable); raxSeek(&ri,"^",NULL,0); while(raxNext(&ri)) { bcastState *bs = ri.data; - /* Create the array reply with the list of keys once, then send - * it to all the clients subscribed to this prefix. */ - char buf[32]; - size_t len = ll2string(buf,sizeof(buf),raxSize(bs->keys)); - sds proto = sdsempty(); - proto = sdsMakeRoomFor(proto,raxSize(bs->keys)*15); - proto = sdscatlen(proto,"*",1); - proto = sdscatlen(proto,buf,len); - proto = sdscatlen(proto,"\r\n",2); - raxStart(&ri2,bs->keys); - raxSeek(&ri2,"^",NULL,0); - while(raxNext(&ri2)) { - len = ll2string(buf,sizeof(buf),ri2.key_len); - sds proto = sdsnewlen("$",1); - proto = sdscatlen(proto,ri2.key,ri2.key_len); + if (raxSize(bs->keys)) { + /* Create the array reply with the list of keys once, then send + * it to all the clients subscribed to this prefix. */ + char buf[32]; + size_t len = ll2string(buf,sizeof(buf),raxSize(bs->keys)); + sds proto = sdsempty(); + proto = sdsMakeRoomFor(proto,raxSize(bs->keys)*15); + proto = sdscatlen(proto,"*",1); + proto = sdscatlen(proto,buf,len); proto = sdscatlen(proto,"\r\n",2); - } - raxStop(&ri2); + raxStart(&ri2,bs->keys); + raxSeek(&ri2,"^",NULL,0); + while(raxNext(&ri2)) { + len = ll2string(buf,sizeof(buf),ri2.key_len); + proto = sdscatlen(proto,"$",1); + proto = sdscatlen(proto,buf,len); + proto = sdscatlen(proto,"\r\n",2); + proto = sdscatlen(proto,ri2.key,ri2.key_len); + proto = sdscatlen(proto,"\r\n",2); + } + raxStop(&ri2); - /* Send this array of keys to every client in the list. */ - raxStart(&ri2,bs->clients); - raxSeek(&ri2,"^",NULL,0); - while(raxNext(&ri2)) { - client *c; - memcpy(&c,ri2.key,sizeof(c)); - sendTrackingMessage(c,proto,sdslen(proto),1); - } - raxStop(&ri2); + /* Send this array of keys to every client in the list. */ + raxStart(&ri2,bs->clients); + raxSeek(&ri2,"^",NULL,0); + while(raxNext(&ri2)) { + client *c; + memcpy(&c,ri2.key,sizeof(c)); + sendTrackingMessage(c,proto,sdslen(proto),1); + } + raxStop(&ri2); - /* Clean up: we can remove everything from this state, because we - * want to only track the new keys that will be accumulated starting - * from now. */ - sdsfree(proto); - raxFree(bs->clients); + /* Clean up: we can remove everything from this state, because we + * want to only track the new keys that will be accumulated starting + * from now. */ + sdsfree(proto); + } raxFree(bs->keys); - bs->clients = raxNew(); bs->keys = raxNew(); } raxStop(&ri); From 3b4f14777569d1f35d20b9b5a7949ecc1803e2d2 Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Thu, 6 Feb 2020 15:06:33 +0200 Subject: [PATCH 063/225] add no-slowlog option to RM_CreateCommand --- src/module.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/module.c b/src/module.c index ed79ee226..aae15d74b 100644 --- a/src/module.c +++ b/src/module.c @@ -730,6 +730,7 @@ int64_t commandFlagsFromString(char *s) { else if (!strcasecmp(t,"random")) flags |= CMD_RANDOM; else if (!strcasecmp(t,"allow-stale")) flags |= CMD_STALE; else if (!strcasecmp(t,"no-monitor")) flags |= CMD_SKIP_MONITOR; + else if (!strcasecmp(t,"no-slowlog")) flags |= CMD_SKIP_SLOWLOG; else if (!strcasecmp(t,"fast")) flags |= CMD_FAST; else if (!strcasecmp(t,"no-auth")) flags |= CMD_NO_AUTH; else if (!strcasecmp(t,"getkeys-api")) flags |= CMD_MODULE_GETKEYS; @@ -781,6 +782,8 @@ int64_t commandFlagsFromString(char *s) { * this means. * * **"no-monitor"**: Don't propagate the command on monitor. Use this if * the command has sensible data among the arguments. + * * **"no-slowlog"**: Don't log this command in the slowlog. Use this if + * the command has sensible data among the arguments. * * **"fast"**: The command time complexity is not greater * than O(log(N)) where N is the size of the collection or * anything else representing the normal scalability From f21be1ec5546cf69b5ec0b492ebcc30143670716 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 13 Feb 2020 16:58:07 +0100 Subject: [PATCH 064/225] Tracking: fix sending messages bug + tracking off bug. --- src/networking.c | 42 ++++++++++++++++++++++-------------------- src/tracking.c | 6 ++++++ 2 files changed, 28 insertions(+), 20 deletions(-) diff --git a/src/networking.c b/src/networking.c index 46534253e..1b4b19645 100644 --- a/src/networking.c +++ b/src/networking.c @@ -2250,37 +2250,39 @@ NULL prefix = zrealloc(prefix,sizeof(robj*)*(numprefix+1)); prefix[numprefix++] = c->argv[j]; } else { + zfree(prefix); addReply(c,shared.syntaxerr); return; } } - /* Make sure options are compatible among each other and with the - * current state of the client. */ - if (!bcast && numprefix) { - addReplyError(c,"PREFIX option requires BCAST mode to be enabled"); - zfree(prefix); - return; - } - - if (c->flags & CLIENT_TRACKING) { - int oldbcast = !!c->flags & CLIENT_TRACKING_BCAST; - if (oldbcast != bcast) { - } - addReplyError(c, - "You can't switch BCAST mode on/off before disabling " - "tracking for this client, and then re-enabling it with " - "a different mode."); - zfree(prefix); - return; - } - /* Options are ok: enable or disable the tracking for this client. */ if (!strcasecmp(c->argv[2]->ptr,"on")) { + /* Before enabling tracking, make sure options are compatible + * among each other and with the current state of the client. */ + if (!bcast && numprefix) { + addReplyError(c, + "PREFIX option requires BCAST mode to be enabled"); + zfree(prefix); + return; + } + + if (c->flags & CLIENT_TRACKING) { + int oldbcast = !!c->flags & CLIENT_TRACKING_BCAST; + if (oldbcast != bcast) { + addReplyError(c, + "You can't switch BCAST mode on/off before disabling " + "tracking for this client, and then re-enabling it with " + "a different mode."); + zfree(prefix); + return; + } + } enableTracking(c,redir,bcast,prefix,numprefix); } else if (!strcasecmp(c->argv[2]->ptr,"off")) { disableTracking(c); } else { + zfree(prefix); addReply(c,shared.syntaxerr); return; } diff --git a/src/tracking.c b/src/tracking.c index 672b886a3..ef5840863 100644 --- a/src/tracking.c +++ b/src/tracking.c @@ -211,6 +211,12 @@ void sendTrackingMessage(client *c, char *keyname, size_t keylen, int proto) { /* We use a static object to speedup things, however we assume * that addReplyPubsubMessage() will not take a reference. */ addReplyPubsubMessage(c,TrackingChannelName,NULL); + } else { + /* If are here, the client is not using RESP3, nor is + * redirecting to another client. We can't send anything to + * it since RESP2 does not support push messages in the same + * connection. */ + return; } /* Send the "value" part, which is the array of keys. */ From 3c16d6b32d0acb31f5ded65df2a12378979acba4 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 7 Feb 2020 14:03:43 +0100 Subject: [PATCH 065/225] Tracking: first conversion from hashing to key names. --- src/server.c | 4 +- src/server.h | 2 +- src/tracking.c | 206 +++++++++++++++++++++---------------------------- 3 files changed, 91 insertions(+), 121 deletions(-) diff --git a/src/server.c b/src/server.c index f87dfba46..df6a8c1f4 100644 --- a/src/server.c +++ b/src/server.c @@ -4221,7 +4221,7 @@ sds genRedisInfoString(const char *section) { "active_defrag_misses:%lld\r\n" "active_defrag_key_hits:%lld\r\n" "active_defrag_key_misses:%lld\r\n" - "tracking_used_slots:%lld\r\n", + "tracking_tracked_keys:%lld\r\n", server.stat_numconnections, server.stat_numcommands, getInstantaneousMetric(STATS_METRIC_COMMAND), @@ -4249,7 +4249,7 @@ sds genRedisInfoString(const char *section) { server.stat_active_defrag_misses, server.stat_active_defrag_key_hits, server.stat_active_defrag_key_misses, - trackingGetUsedSlots()); + (unsigned long long) trackingGetTotalItems()); } /* Replication */ diff --git a/src/server.h b/src/server.h index dfbc15ac3..a30db6b57 100644 --- a/src/server.h +++ b/src/server.h @@ -1654,7 +1654,7 @@ void trackingRememberKeys(client *c); void trackingInvalidateKey(robj *keyobj); void trackingInvalidateKeysOnFlush(int dbid); void trackingLimitUsedSlots(void); -unsigned long long trackingGetUsedSlots(void); +uint64_t trackingGetTotalItems(void); /* List data type */ void listTypeTryConversion(robj *subject, robj *value); diff --git a/src/tracking.c b/src/tracking.c index acb97800a..ecb7fbdcc 100644 --- a/src/tracking.c +++ b/src/tracking.c @@ -30,37 +30,22 @@ #include "server.h" -/* The tracking table is constituted by 2^24 radix trees (each tree, and the - * table itself, are allocated in a lazy way only when needed) tracking - * clients that may have certain keys in their local, client side, cache. - * - * Keys are grouped into 2^24 slots, in a way similar to Redis Cluster hash - * slots, however here the function we use is crc64, taking the least - * significant 24 bits of the output. +/* The tracking table is constituted by a radix tree of keys, each pointing + * to a radix tree of client IDs, used to track the clients that may have + * certain keys in their local, client side, cache. * * When a client enables tracking with "CLIENT TRACKING on", each key served to - * the client is hashed to one of such slots, and Redis will remember what - * client may have keys about such slot. Later, when a key in a given slot is - * modified, all the clients that may have local copies of keys in that slot - * will receive an invalidation message. There is no distinction of database - * number: a single table is used. + * the client is remembered in the table mapping the keys to the client IDs. + * Later, when a key is modified, all the clients that may have local copy + * of such key will receive an invalidation message. * * Clients will normally take frequently requested objects in memory, removing - * them when invalidation messages are received. A strategy clients may use is - * to just cache objects in a dictionary, associating to each cached object - * some incremental epoch, or just a timestamp. When invalidation messages are - * received clients may store, in a different table, the timestamp (or epoch) - * of the invalidation of such given slot: later when accessing objects, the - * eviction of stale objects may be performed in a lazy way by checking if the - * cached object timestamp is older than the invalidation timestamp for such - * objects. - * - * The output of the 24 bit hash function is very large (more than 16 million - * possible slots), so clients that may want to use less resources may only - * use the most significant bits instead of the full 24 bits. */ -#define TRACKING_TABLE_SIZE (1<<24) -rax **TrackingTable = NULL; -unsigned long TrackingTableUsedSlots = 0; + * them when invalidation messages are received. */ +rax *TrackingTable = NULL; +uint64_t TrackingTableTotalItems = 0; /* Total number of IDs stored across + the whole tracking table. This givesn + an hint about the total memory we + are using server side for CSC. */ robj *TrackingChannelName; /* Remove the tracking state from the client 'c'. Note that there is not much @@ -90,7 +75,7 @@ void enableTracking(client *c, uint64_t redirect_to) { c->client_tracking_redirection = redirect_to; server.tracking_clients++; if (TrackingTable == NULL) { - TrackingTable = zcalloc(sizeof(rax*) * TRACKING_TABLE_SIZE); + TrackingTable = raxNew(); TrackingChannelName = createStringObject("__redis__:invalidate",20); } } @@ -108,19 +93,20 @@ void trackingRememberKeys(client *c) { for(int j = 0; j < numkeys; j++) { int idx = keys[j]; sds sdskey = c->argv[idx]->ptr; - uint64_t hash = crc64(0, - (unsigned char*)sdskey,sdslen(sdskey))&(TRACKING_TABLE_SIZE-1); - if (TrackingTable[hash] == NULL) { - TrackingTable[hash] = raxNew(); - TrackingTableUsedSlots++; + rax *ids = raxFind(TrackingTable,(unsigned char*)sdskey,sdslen(sdskey)); + if (ids == raxNotFound) { + ids = raxNew(); + int inserted = raxTryInsert(TrackingTable,(unsigned char*)sdskey, + sdslen(sdskey),ids, NULL); + serverAssert(inserted == 1); } - raxTryInsert(TrackingTable[hash], - (unsigned char*)&c->id,sizeof(c->id),NULL,NULL); + if (raxTryInsert(ids,(unsigned char*)&c->id,sizeof(c->id),NULL,NULL)) + TrackingTableTotalItems++; } getKeysFreeResult(keys); } -void sendTrackingMessage(client *c, long long hash) { +void sendTrackingMessage(client *c, char *keyname, size_t keylen) { int using_redirection = 0; if (c->client_tracking_redirection) { client *redir = lookupClientByID(c->client_tracking_redirection); @@ -146,49 +132,44 @@ void sendTrackingMessage(client *c, long long hash) { if (c->resp > 2) { addReplyPushLen(c,2); addReplyBulkCBuffer(c,"invalidate",10); - addReplyLongLong(c,hash); + addReplyBulkCBuffer(c,keyname,keylen); } else if (using_redirection && c->flags & CLIENT_PUBSUB) { - robj *msg = createStringObjectFromLongLong(hash); - addReplyPubsubMessage(c,TrackingChannelName,msg); - decrRefCount(msg); + /* We use a static object to speedup things, however we assume + * that addReplyPubsubMessage() will not take a reference. */ + robj keyobj; + initStaticStringObject(keyobj,keyname); + addReplyPubsubMessage(c,TrackingChannelName,&keyobj); + serverAssert(keyobj.refcount == 1); } } -/* Invalidates a caching slot: this is actually the low level implementation - * of the API that Redis calls externally, that is trackingInvalidateKey(). */ -void trackingInvalidateSlot(uint64_t slot) { - if (TrackingTable == NULL || TrackingTable[slot] == NULL) return; - - raxIterator ri; - raxStart(&ri,TrackingTable[slot]); - raxSeek(&ri,"^",NULL,0); - while(raxNext(&ri)) { - uint64_t id; - memcpy(&id,ri.key,sizeof(id)); - client *c = lookupClientByID(id); - if (c == NULL || !(c->flags & CLIENT_TRACKING)) continue; - sendTrackingMessage(c,slot); - } - raxStop(&ri); - - /* Free the tracking table: we'll create the radix tree and populate it - * again if more keys will be modified in this caching slot. */ - raxFree(TrackingTable[slot]); - TrackingTable[slot] = NULL; - TrackingTableUsedSlots--; -} - /* This function is called from signalModifiedKey() or other places in Redis * when a key changes value. In the context of keys tracking, our task here is * to send a notification to every client that may have keys about such caching * slot. */ void trackingInvalidateKey(robj *keyobj) { - if (TrackingTable == NULL || TrackingTableUsedSlots == 0) return; - + if (TrackingTable == NULL) return; sds sdskey = keyobj->ptr; - uint64_t hash = crc64(0, - (unsigned char*)sdskey,sdslen(sdskey))&(TRACKING_TABLE_SIZE-1); - trackingInvalidateSlot(hash); + rax *ids = raxFind(TrackingTable,(unsigned char*)sdskey,sdslen(sdskey)); + if (ids == raxNotFound) return;; + + raxIterator ri; + raxStart(&ri,ids); + raxSeek(&ri,"^",NULL,0); + while(raxNext(&ri)) { + uint64_t id; + memcpy(&id,ri.key,sizeof(id)); + client *c = lookupClientByID(id); + if (c == NULL || !(c->flags & CLIENT_TRACKING)) continue; + sendTrackingMessage(c,sdskey,sdslen(sdskey)); + } + raxStop(&ri); + + /* Free the tracking table: we'll create the radix tree and populate it + * again if more keys will be modified in this caching slot. */ + TrackingTableTotalItems -= raxSize(ids); + raxFree(ids); + raxRemove(TrackingTable,(unsigned char*)sdskey,sdslen(sdskey),NULL); } /* This function is called when one or all the Redis databases are flushed @@ -205,6 +186,10 @@ void trackingInvalidateKey(robj *keyobj) { * we just send the invalidation message to all the clients, but don't * flush the table: it will slowly get garbage collected as more keys * are modified in the used caching slots. */ +void freeTrackingRadixTree(void *rt) { + raxFree(rt); +} + void trackingInvalidateKeysOnFlush(int dbid) { if (server.tracking_clients) { listNode *ln; @@ -213,84 +198,69 @@ void trackingInvalidateKeysOnFlush(int dbid) { while ((ln = listNext(&li)) != NULL) { client *c = listNodeValue(ln); if (c->flags & CLIENT_TRACKING) { - sendTrackingMessage(c,-1); + sendTrackingMessage(c,"",1); } } } /* In case of FLUSHALL, reclaim all the memory used by tracking. */ if (dbid == -1 && TrackingTable) { - for (int j = 0; j < TRACKING_TABLE_SIZE && TrackingTableUsedSlots > 0; j++) { - if (TrackingTable[j] != NULL) { - raxFree(TrackingTable[j]); - TrackingTable[j] = NULL; - TrackingTableUsedSlots--; - } - } - - /* If there are no clients with tracking enabled, we can even - * reclaim the memory used by the table itself. The code assumes - * the table is allocated only if there is at least one client alive - * with tracking enabled. */ - if (server.tracking_clients == 0) { - zfree(TrackingTable); - TrackingTable = NULL; - } + raxFreeWithCallback(TrackingTable,freeTrackingRadixTree); + TrackingTableTotalItems = 0; } } /* Tracking forces Redis to remember information about which client may have - * keys about certian caching slots. In workloads where there are a lot of - * reads, but keys are hardly modified, the amount of information we have - * to remember server side could be a lot: for each 16 millions of caching - * slots we may end with a radix tree containing many entries. + * certain keys. In workloads where there are a lot of reads, but keys are + * hardly modified, the amount of information we have to remember server side + * could be a lot, with the number of keys being totally not bound. * - * So Redis allows the user to configure a maximum fill rate for the + * So Redis allows the user to configure a maximum number of keys for the * invalidation table. This function makes sure that we don't go over the * specified fill rate: if we are over, we can just evict informations about - * random caching slots, and send invalidation messages to clients like if - * the key was modified. */ + * a random key, and send invalidation messages to clients like if the key was + * modified. */ void trackingLimitUsedSlots(void) { static unsigned int timeout_counter = 0; - + if (TrackingTable == NULL) return; if (server.tracking_table_max_fill == 0) return; /* No limits set. */ - unsigned int max_slots = - (TRACKING_TABLE_SIZE/100) * server.tracking_table_max_fill; - if (TrackingTableUsedSlots <= max_slots) { + size_t max_keys = server.tracking_table_max_fill; + if (raxSize(TrackingTable) <= max_keys) { timeout_counter = 0; return; /* Limit not reached. */ } - /* We have to invalidate a few slots to reach the limit again. The effort + /* We have to invalidate a few keys to reach the limit again. The effort * we do here is proportional to the number of times we entered this * function and found that we are still over the limit. */ int effort = 100 * (timeout_counter+1); - /* Let's start at a random position, and perform linear probing, in order - * to improve cache locality. However once we are able to find an used - * slot, jump again randomly, in order to avoid creating big holes in the - * table (that will make this funciton use more resourced later). */ + /* We just remove one key after another by using a random walk. */ + raxIterator ri; + raxStart(&ri,TrackingTable); while(effort > 0) { - unsigned int idx = rand() % TRACKING_TABLE_SIZE; - do { - effort--; - idx = (idx+1) % TRACKING_TABLE_SIZE; - if (TrackingTable[idx] != NULL) { - trackingInvalidateSlot(idx); - if (TrackingTableUsedSlots <= max_slots) { - timeout_counter = 0; - return; /* Return ASAP: we are again under the limit. */ - } else { - break; /* Jump to next random position. */ - } - } - } while(effort > 0); + effort--; + raxSeek(&ri,"^",NULL,0); + raxRandomWalk(&ri,0); + rax *ids = ri.data; + TrackingTableTotalItems -= raxSize(ids); + raxFree(ids); + raxRemove(TrackingTable,ri.key,ri.key_len,NULL); + if (raxSize(TrackingTable) <= max_keys) { + timeout_counter = 0; + raxStop(&ri); + return; /* Return ASAP: we are again under the limit. */ + } } + + /* If we reach this point, we were not able to go under the configured + * limit using the maximum effort we had for this run. */ + raxStop(&ri); timeout_counter++; } /* This is just used in order to access the amount of used slots in the * tracking table. */ -unsigned long long trackingGetUsedSlots(void) { - return TrackingTableUsedSlots; +uint64_t trackingGetTotalItems(void) { + return TrackingTableTotalItems; } From fe96e29d5ff4deed5a1b95181e4eae4e1862a3bd Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 14 Feb 2020 14:17:10 +0100 Subject: [PATCH 066/225] Tracking: fix behavior when switchinig from normal to BCAST. --- src/tracking.c | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/tracking.c b/src/tracking.c index ef5840863..3333472a4 100644 --- a/src/tracking.c +++ b/src/tracking.c @@ -269,7 +269,17 @@ void trackingInvalidateKey(robj *keyobj) { uint64_t id; memcpy(&id,ri.key,sizeof(id)); client *c = lookupClientByID(id); - if (c == NULL || !(c->flags & CLIENT_TRACKING)) continue; + /* Note that if the client is in BCAST mode, we don't want to + * send invalidation messages that were pending in the case + * previously the client was not in BCAST mode. This can happen if + * TRACKING is enabled normally, and then the client switches to + * BCAST mode. */ + if (c == NULL || + !(c->flags & CLIENT_TRACKING)|| + c->flags & CLIENT_TRACKING_BCAST) + { + continue; + } sendTrackingMessage(c,sdskey,sdslen(sdskey),0); } raxStop(&ri); From 0517da3618be3af60d5e928f983a437d3172ff82 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 7 Feb 2020 17:19:11 +0100 Subject: [PATCH 067/225] Tracking: rename INFO field with total items. --- src/server.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/server.c b/src/server.c index df6a8c1f4..2cf8dbf19 100644 --- a/src/server.c +++ b/src/server.c @@ -4221,7 +4221,7 @@ sds genRedisInfoString(const char *section) { "active_defrag_misses:%lld\r\n" "active_defrag_key_hits:%lld\r\n" "active_defrag_key_misses:%lld\r\n" - "tracking_tracked_keys:%lld\r\n", + "tracking_total_items:%lld\r\n", server.stat_numconnections, server.stat_numcommands, getInstantaneousMetric(STATS_METRIC_COMMAND), From 1db72571074860245573954e49878dc67ec70077 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 14 Feb 2020 14:27:22 +0100 Subject: [PATCH 068/225] Tracking: fix operators precedence error in bcast check. --- src/networking.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/networking.c b/src/networking.c index 1b4b19645..69350eed1 100644 --- a/src/networking.c +++ b/src/networking.c @@ -2268,7 +2268,7 @@ NULL } if (c->flags & CLIENT_TRACKING) { - int oldbcast = !!c->flags & CLIENT_TRACKING_BCAST; + int oldbcast = !!(c->flags & CLIENT_TRACKING_BCAST); if (oldbcast != bcast) { addReplyError(c, "You can't switch BCAST mode on/off before disabling " From df83892760d19b1b7e92d76e72daf4834ad2df6c Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 7 Feb 2020 18:12:10 +0100 Subject: [PATCH 069/225] Rax.c: populate data field after random walk. --- src/rax.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/rax.c b/src/rax.c index 29b74ae90..a560dde02 100644 --- a/src/rax.c +++ b/src/rax.c @@ -1766,6 +1766,7 @@ int raxRandomWalk(raxIterator *it, size_t steps) { if (n->iskey) steps--; } it->node = n; + it->data = raxGetData(it->node); return 1; } From b7cb28d5011762a1ac062735b33e9b946957fb87 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 14 Feb 2020 14:29:00 +0100 Subject: [PATCH 070/225] Tracking: first set of tests for the feature. --- tests/unit/tracking.tcl | 66 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 tests/unit/tracking.tcl diff --git a/tests/unit/tracking.tcl b/tests/unit/tracking.tcl new file mode 100644 index 000000000..2058319f7 --- /dev/null +++ b/tests/unit/tracking.tcl @@ -0,0 +1,66 @@ +start_server {tags {"tracking"}} { + # Create a deferred client we'll use to redirect invalidation + # messages to. + set rd1 [redis_deferring_client] + $rd1 client id + set redir [$rd1 read] + $rd1 subscribe __redis__:invalidate + $rd1 read ; # Consume the SUBSCRIBE reply. + + test {Clients are able to enable tracking and redirect it} { + r CLIENT TRACKING on REDIRECT $redir + } {*OK} + + test {The other connection is able to get invalidations} { + r SET a 1 + r GET a + r INCR a + r INCR b ; # This key should not be notified, since it wasn't fetched. + set keys [lindex [$rd1 read] 2] + assert {[llength $keys] == 1} + assert {[lindex $keys 0] eq {a}} + } + + test {The client is now able to disable tracking} { + # Make sure to add a few more keys in the tracking list + # so that we can check for leaks, as a side effect. + r MGET a b c d e f g + r CLIENT TRACKING off + } + + test {Clients can enable the BCAST mode with the empty prefix} { + r CLIENT TRACKING on BCAST REDIRECT $redir + } {*OK*} + + test {The connection gets invalidation messages about all the keys} { + r MSET a 1 b 2 c 3 + set keys [lsort [lindex [$rd1 read] 2]] + assert {$keys eq {a b c}} + } + + test {Clients can enable the BCAST mode with prefixes} { + r CLIENT TRACKING off + r CLIENT TRACKING on BCAST REDIRECT $redir PREFIX a: PREFIX b: + r MULTI + r INCR a:1 + r INCR a:2 + r INCR b:1 + r INCR b:2 + r EXEC + # Because of the internals, we know we are going to receive + # two separated notifications for the two different prefixes. + set keys1 [lsort [lindex [$rd1 read] 2]] + set keys2 [lsort [lindex [$rd1 read] 2]] + set keys [lsort [list {*}$keys1 {*}$keys2]] + assert {$keys eq {a:1 a:2 b:1 b:2}} + } + + test {Adding prefixes to BCAST mode works} { + r CLIENT TRACKING on BCAST REDIRECT $redir PREFIX c: + r INCR c:1234 + set keys [lsort [lindex [$rd1 read] 2]] + assert {$keys eq {c:1234}} + } + + $rd1 close +} From 73d47d574946796bcb2fe9330d5062971ad56852 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 14 Feb 2020 18:22:25 +0100 Subject: [PATCH 071/225] Signal key as modified when expired on-access. This fixes WATCH and client side caching with keys expiring because of a synchronous access and not because of background expiring. --- src/db.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/db.c b/src/db.c index c80524e94..8a0242d9e 100644 --- a/src/db.c +++ b/src/db.c @@ -1296,8 +1296,10 @@ int expireIfNeeded(redisDb *db, robj *key) { propagateExpire(db,key,server.lazyfree_lazy_expire); notifyKeyspaceEvent(NOTIFY_EXPIRED, "expired",key,db->id); - return server.lazyfree_lazy_expire ? dbAsyncDelete(db,key) : - dbSyncDelete(db,key); + int retval = server.lazyfree_lazy_expire ? dbAsyncDelete(db,key) : + dbSyncDelete(db,key); + if (retval) signalModifiedKey(db,key); + return retval; } /* ----------------------------------------------------------------------------- From e374573f301ccecc36ced08330d4f19d9e097d23 Mon Sep 17 00:00:00 2001 From: Itamar Haber Date: Fri, 14 Feb 2020 17:13:58 +0200 Subject: [PATCH 072/225] Fixes segfault on calling trackingGetTotalKeys ... with CSC disabled --- src/tracking.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/tracking.c b/src/tracking.c index 3333472a4..7179a54f8 100644 --- a/src/tracking.c +++ b/src/tracking.c @@ -441,5 +441,6 @@ uint64_t trackingGetTotalItems(void) { } uint64_t trackingGetTotalKeys(void) { + if (TrackingTable == NULL) return 0; return raxSize(TrackingTable); } From f15fb727a0dae31c30befe1a6f536cbbbf9f04e2 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 19 Feb 2020 19:00:29 +0100 Subject: [PATCH 073/225] Tracking: fix max-keys configuration directive. --- src/config.c | 2 +- src/server.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/config.c b/src/config.c index b2e5fc12e..d55d1f8b5 100644 --- a/src/config.c +++ b/src/config.c @@ -2160,7 +2160,6 @@ standardConfig configs[] = { createIntConfig("list-compress-depth", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.list_compress_depth, 0, INTEGER_CONFIG, NULL, NULL), createIntConfig("rdb-key-save-delay", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.rdb_key_save_delay, 0, INTEGER_CONFIG, NULL, NULL), createIntConfig("key-load-delay", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.key_load_delay, 0, INTEGER_CONFIG, NULL, NULL), - createIntConfig("tracking-table-max-fill", NULL, MODIFIABLE_CONFIG, 0, 100, server.tracking_table_max_keys, 1000000, INTEGER_CONFIG, NULL, NULL), /* Default: 10% tracking table max number of keys tracked. */ createIntConfig("active-expire-effort", NULL, MODIFIABLE_CONFIG, 1, 10, server.active_expire_effort, 1, INTEGER_CONFIG, NULL, NULL), /* From 1 to 10. */ createIntConfig("hz", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.config_hz, CONFIG_DEFAULT_HZ, INTEGER_CONFIG, NULL, updateHZ), createIntConfig("min-replicas-to-write", "min-slaves-to-write", MODIFIABLE_CONFIG, 0, INT_MAX, server.repl_min_slaves_to_write, 0, INTEGER_CONFIG, NULL, updateGoodSlaves), @@ -2195,6 +2194,7 @@ standardConfig configs[] = { createSizeTConfig("stream-node-max-bytes", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.stream_node_max_bytes, 4096, MEMORY_CONFIG, NULL, NULL), createSizeTConfig("zset-max-ziplist-value", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.zset_max_ziplist_value, 64, MEMORY_CONFIG, NULL, NULL), createSizeTConfig("hll-sparse-max-bytes", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.hll_sparse_max_bytes, 3000, MEMORY_CONFIG, NULL, NULL), + createSizeTConfig("tracking-table-max-keys", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.tracking_table_max_keys, 1000000, INTEGER_CONFIG, NULL, NULL), /* Default: 1 million keys max. */ /* Other configs */ createTimeTConfig("repl-backlog-ttl", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.repl_backlog_time_limit, 60*60, INTEGER_CONFIG, NULL, NULL), /* Default: 1 hour */ diff --git a/src/server.h b/src/server.h index 439bbc393..bb40ca4e1 100644 --- a/src/server.h +++ b/src/server.h @@ -1309,7 +1309,7 @@ struct redisServer { list *ready_keys; /* List of readyList structures for BLPOP & co */ /* Client side caching. */ unsigned int tracking_clients; /* # of clients with tracking enabled.*/ - int tracking_table_max_keys; /* Max number of keys in tracking table. */ + size_t tracking_table_max_keys; /* Max number of keys in tracking table. */ /* Sort parameters - qsort_r() is only available under BSD so we * have to take this state global, in order to pass it to sortCompare() */ int sort_desc; From 762fbcb687c6823d3403cfe1ea485aced0ba4409 Mon Sep 17 00:00:00 2001 From: Madelyn Olson Date: Sun, 16 Feb 2020 05:16:51 -0800 Subject: [PATCH 074/225] Minor CSC fixes and fixed documentation --- src/networking.c | 11 ++++++++--- src/tracking.c | 22 ++++++++-------------- 2 files changed, 16 insertions(+), 17 deletions(-) diff --git a/src/networking.c b/src/networking.c index 69350eed1..dad61904d 100644 --- a/src/networking.c +++ b/src/networking.c @@ -2044,7 +2044,7 @@ void clientCommand(client *c) { "REPLY (on|off|skip) -- Control the replies sent to the current connection.", "SETNAME -- Assign the name to the current connection.", "UNBLOCK [TIMEOUT|ERROR] -- Unblock the specified blocked client.", -"TRACKING (on|off) [REDIRECT ] -- Enable client keys tracking for client side caching.", +"TRACKING (on|off) [REDIRECT ] [BCAST] [PREFIX first] [PREFIX second] ... -- Enable client keys tracking for client side caching.", "GETREDIR -- Return the client ID we are redirecting to when tracking is enabled.", NULL }; @@ -2234,17 +2234,22 @@ NULL if (!strcasecmp(c->argv[j]->ptr,"redirect") && moreargs) { j++; if (getLongLongFromObjectOrReply(c,c->argv[j],&redir,NULL) != - C_OK) return; + C_OK) + { + zfree(prefix); + return; + } /* We will require the client with the specified ID to exist * right now, even if it is possible that it gets disconnected * later. Still a valid sanity check. */ if (lookupClientByID(redir) == NULL) { addReplyError(c,"The client ID you want redirect to " "does not exist"); + zfree(prefix); return; } } else if (!strcasecmp(c->argv[j]->ptr,"bcast")) { - bcast++; + bcast = 1; } else if (!strcasecmp(c->argv[j]->ptr,"prefix") && moreargs) { j++; prefix = zrealloc(prefix,sizeof(robj*)*(numprefix+1)); diff --git a/src/tracking.c b/src/tracking.c index 7179a54f8..619148f2f 100644 --- a/src/tracking.c +++ b/src/tracking.c @@ -44,7 +44,7 @@ rax *TrackingTable = NULL; rax *PrefixTable = NULL; uint64_t TrackingTableTotalItems = 0; /* Total number of IDs stored across - the whole tracking table. This givesn + the whole tracking table. This gives an hint about the total memory we are using server side for CSC. */ robj *TrackingChannelName; @@ -145,9 +145,9 @@ void enableTracking(client *c, uint64_t redirect_to, int bcast, robj **prefix, s } } -/* This function is called after the excution of a readonly command in the +/* This function is called after the execution of a readonly command in the * case the client 'c' has keys tracking enabled. It will populate the - * tracking ivalidation table according to the keys the user fetched, so that + * tracking invalidation table according to the keys the user fetched, so that * Redis will know what are the clients that should receive an invalidation * message with certain groups of keys are modified. */ void trackingRememberKeys(client *c) { @@ -292,19 +292,12 @@ void trackingInvalidateKey(robj *keyobj) { } /* This function is called when one or all the Redis databases are flushed - * (dbid == -1 in case of FLUSHALL). Caching slots are not specific for - * each DB but are global: currently what we do is sending a special + * (dbid == -1 in case of FLUSHALL). Caching keys are not specific for + * each DB but are global: currently what we do is send a special * notification to clients with tracking enabled, invalidating the caching - * slot "-1", which means, "all the keys", in order to avoid flooding clients + * key "", which means, "all the keys", in order to avoid flooding clients * with many invalidation messages for all the keys they may hold. - * - * However trying to flush the tracking table here is very costly: - * we need scanning 16 million caching slots in the table to check - * if they are used, this introduces a big delay. So what we do is to really - * flush the table in the case of FLUSHALL. When a FLUSHDB is called instead - * we just send the invalidation message to all the clients, but don't - * flush the table: it will slowly get garbage collected as more keys - * are modified in the used caching slots. */ + */ void freeTrackingRadixTree(void *rt) { raxFree(rt); } @@ -325,6 +318,7 @@ void trackingInvalidateKeysOnFlush(int dbid) { /* In case of FLUSHALL, reclaim all the memory used by tracking. */ if (dbid == -1 && TrackingTable) { raxFreeWithCallback(TrackingTable,freeTrackingRadixTree); + TrackingTable = raxNew(); TrackingTableTotalItems = 0; } } From d1f22eaca422039e89b593e8c951439a538a4764 Mon Sep 17 00:00:00 2001 From: Madelyn Olson Date: Sun, 16 Feb 2020 05:41:39 -0800 Subject: [PATCH 075/225] Give an error message if you specify redirect twice --- src/networking.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/networking.c b/src/networking.c index dad61904d..5b1229fde 100644 --- a/src/networking.c +++ b/src/networking.c @@ -2233,6 +2233,13 @@ NULL if (!strcasecmp(c->argv[j]->ptr,"redirect") && moreargs) { j++; + if (redir != 0) { + addReplyError(c,"A client can only redirect to a single " + "other client"); + zfree(prefix); + return; + } + if (getLongLongFromObjectOrReply(c,c->argv[j],&redir,NULL) != C_OK) { From b6129f86c1468b9c0331d654fb5cc5ba0a1c6de2 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 21 Feb 2020 13:48:43 +0100 Subject: [PATCH 076/225] Test is more complex now, increase default timeout. --- tests/test_helper.tcl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_helper.tcl b/tests/test_helper.tcl index cb7e4e328..b266bc56d 100644 --- a/tests/test_helper.tcl +++ b/tests/test_helper.tcl @@ -87,7 +87,7 @@ set ::file ""; # If set, runs only the tests in this comma separated list set ::curfile ""; # Hold the filename of the current suite set ::accurate 0; # If true runs fuzz tests with more iterations set ::force_failure 0 -set ::timeout 600; # 10 minutes without progresses will quit the test. +set ::timeout 1200; # 20 minutes without progresses will quit the test. set ::last_progress [clock seconds] set ::active_servers {} ; # Pids of active Redis instances. set ::dont_clean 0 From 5d0890c0a83e4f3c2398a3ff7381c72ef36602df Mon Sep 17 00:00:00 2001 From: Guy Benoish Date: Thu, 6 Feb 2020 18:36:21 +0530 Subject: [PATCH 077/225] Fix memory leak in test_ld_conv --- tests/modules/misc.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/modules/misc.c b/tests/modules/misc.c index 41bec06ed..1048d5065 100644 --- a/tests/modules/misc.c +++ b/tests/modules/misc.c @@ -74,6 +74,7 @@ int test_ld_conv(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { RedisModule_ReplyWithError(ctx, err); goto final; } + /* Make sure we can't convert a string that has \0 in it */ char buf[4] = "123"; buf[1] = '\0'; @@ -81,8 +82,11 @@ int test_ld_conv(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { long double ld3; if (RedisModule_StringToLongDouble(s3, &ld3) == REDISMODULE_OK) { RedisModule_ReplyWithError(ctx, "Invalid string successfully converted to long double"); + RedisModule_FreeString(ctx, s3); goto final; } + RedisModule_FreeString(ctx, s3); + RedisModule_ReplyWithLongDouble(ctx, ld2); final: RedisModule_FreeString(ctx, s1); From 73806f74cac3c524f1824197500f43135bc1c3cf Mon Sep 17 00:00:00 2001 From: hayashier Date: Tue, 31 Dec 2019 17:46:48 +0900 Subject: [PATCH 078/225] fix typo from fss to rss --- src/object.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/object.c b/src/object.c index cc6b218a0..11e335afc 100644 --- a/src/object.c +++ b/src/object.c @@ -1102,13 +1102,13 @@ sds getMemoryDoctorReport(void) { num_reports++; } - /* Allocator fss is higher than 1.1 and 10MB ? */ + /* Allocator rss is higher than 1.1 and 10MB ? */ if (mh->allocator_rss > 1.1 && mh->allocator_rss_bytes > 10<<20) { high_alloc_rss = 1; num_reports++; } - /* Non-Allocator fss is higher than 1.1 and 10MB ? */ + /* Non-Allocator rss is higher than 1.1 and 10MB ? */ if (mh->rss_extra > 1.1 && mh->rss_extra_bytes > 10<<20) { high_proc_rss = 1; num_reports++; From ba0270799ed6b347c5e096a5ebbf9c7e1bc5c370 Mon Sep 17 00:00:00 2001 From: hwware Date: Mon, 17 Feb 2020 23:40:24 -0500 Subject: [PATCH 079/225] add missing subcommand description for debug oom --- src/debug.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/debug.c b/src/debug.c index b910d2d2d..dd96ad416 100644 --- a/src/debug.c +++ b/src/debug.c @@ -363,6 +363,7 @@ void debugCommand(client *c) { "LOADAOF -- Flush the AOF buffers on disk and reload the AOF in memory.", "LUA-ALWAYS-REPLICATE-COMMANDS <0|1> -- Setting it to 1 makes Lua replication defaulting to replicating single commands, without the script having to enable effects replication.", "OBJECT -- Show low level info about key and associated value.", +"OOM -- Crash the server simulating an out-of-memory error.", "PANIC -- Crash the server simulating a panic.", "POPULATE [prefix] [size] -- Create string keys named key:. If a prefix is specified is used instead of the 'key' prefix.", "RELOAD -- Save the RDB on disk and reload it back in memory.", From b4ddc7b7bab88dfa730662280125fa3688c2b78c Mon Sep 17 00:00:00 2001 From: Guy Benoish Date: Wed, 19 Feb 2020 08:24:20 +0530 Subject: [PATCH 080/225] XGROUP DESTROY should unblock XREADGROUP with -NOGROUP --- src/t_stream.c | 2 ++ tests/unit/type/stream-cgroups.tcl | 9 +++++++++ 2 files changed, 11 insertions(+) diff --git a/src/t_stream.c b/src/t_stream.c index 0f0f97a1d..900fa3a17 100644 --- a/src/t_stream.c +++ b/src/t_stream.c @@ -1850,6 +1850,8 @@ NULL server.dirty++; notifyKeyspaceEvent(NOTIFY_STREAM,"xgroup-destroy", c->argv[2],c->db->id); + /* We want to unblock any XREADGROUP consumers with -NOGROUP. */ + signalKeyAsReady(c->db,c->argv[2]); } else { addReply(c,shared.czero); } diff --git a/tests/unit/type/stream-cgroups.tcl b/tests/unit/type/stream-cgroups.tcl index a59e168ef..072ed14d6 100644 --- a/tests/unit/type/stream-cgroups.tcl +++ b/tests/unit/type/stream-cgroups.tcl @@ -161,6 +161,15 @@ start_server { assert {[$rd read] == {}} ;# before the fix, client didn't even block, but was served synchronously with {mystream {}} } + test {XGROUP DESTROY should unblock XREADGROUP with -NOGROUP} { + r del mystream + r XGROUP CREATE mystream mygroup $ MKSTREAM + set rd [redis_deferring_client] + $rd XREADGROUP GROUP mygroup Alice BLOCK 100 STREAMS mystream ">" + r XGROUP DESTROY mystream mygroup + assert_error "*NOGROUP*" {$rd read} + } + test {XCLAIM can claim PEL items from another consumer} { # Add 3 items into the stream, and create a consumer group r del mystream From 294c9af469b1bc883dc5b8d6d6b8ce7ec695f29a Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 21 Feb 2020 17:08:45 +0100 Subject: [PATCH 081/225] Test engine: better tracking of what workers are doing. --- tests/support/server.tcl | 3 +++ tests/test_helper.tcl | 13 +++++++++---- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/tests/support/server.tcl b/tests/support/server.tcl index b20f1ad36..174b05852 100644 --- a/tests/support/server.tcl +++ b/tests/support/server.tcl @@ -53,6 +53,7 @@ proc kill_server config { } # kill server and wait for the process to be totally exited + send_data_packet $::test_server_fd server-killing $pid catch {exec kill $pid} if {$::valgrind} { set max_wait 60000 @@ -231,6 +232,8 @@ proc start_server {options {code undefined}} { set stdout [format "%s/%s" [dict get $config "dir"] "stdout"] set stderr [format "%s/%s" [dict get $config "dir"] "stderr"] + send_data_packet $::test_server_fd "server-spawning" "port $::port" + if {$::valgrind} { set pid [exec valgrind --track-origins=yes --suppressions=src/valgrind.sup --show-reachable=no --show-possibly-lost=no --leak-check=full src/redis-server $config_file > $stdout 2> $stderr &] } elseif ($::stack_logging) { diff --git a/tests/test_helper.tcl b/tests/test_helper.tcl index b266bc56d..fa5579669 100644 --- a/tests/test_helper.tcl +++ b/tests/test_helper.tcl @@ -289,7 +289,7 @@ proc read_from_test_client fd { puts "\[$completed_tests_count/$all_tests_count [colorstr yellow $status]\]: $data ($elapsed seconds)" lappend ::clients_time_history $elapsed $data signal_idle_client $fd - set ::active_clients_task($fd) DONE + set ::active_clients_task($fd) "(DONE) $data" } elseif {$status eq {ok}} { if {!$::quiet} { puts "\[[colorstr green $status]\]: $data" @@ -320,10 +320,16 @@ proc read_from_test_client fd { exit 1 } elseif {$status eq {testing}} { set ::active_clients_task($fd) "(IN PROGRESS) $data" + } elseif {$status eq {server-spawning}} { + set ::active_clients_task($fd) "(SPAWNING SERVER) $data" } elseif {$status eq {server-spawned}} { lappend ::active_servers $data + set ::active_clients_task($fd) "(SPAWNED SERVER) pid:$data" + } elseif {$status eq {server-killing}} { + set ::active_clients_task($fd) "(KILLING SERVER) pid:$data" } elseif {$status eq {server-killed}} { set ::active_servers [lsearch -all -inline -not -exact $::active_servers $data] + set ::active_clients_task($fd) "(KILLED SERVER) pid:$data" } else { if {!$::quiet} { puts "\[$status\]: $data" @@ -333,7 +339,7 @@ proc read_from_test_client fd { proc show_clients_state {} { # The following loop is only useful for debugging tests that may - # enter an infinite loop. Commented out normally. + # enter an infinite loop. foreach x $::active_clients { if {[info exist ::active_clients_task($x)]} { puts "$x => $::active_clients_task($x)" @@ -363,8 +369,6 @@ proc signal_idle_client fd { set ::active_clients \ [lsearch -all -inline -not -exact $::active_clients $fd] - if 0 {show_clients_state} - # New unit to process? if {$::next_test != [llength $::all_tests]} { if {!$::quiet} { @@ -380,6 +384,7 @@ proc signal_idle_client fd { } } else { lappend ::idle_clients $fd + set ::active_clients_task($fd) "SLEEPING, no more units to assign" if {[llength $::active_clients] == 0} { the_end } From 349aa24511253b3e323064b95cefb4414f31726c Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Tue, 18 Feb 2020 16:19:52 +0200 Subject: [PATCH 082/225] Defrag big lists in portions to avoid latency and freeze When active defrag kicks in and finds a big list, it will create a bookmark to a node so that it is able to resume iteration from that node later. The quicklist manages that bookmark, and updates it in case that node is deleted. This will increase memory usage only on lists of over 1000 (see active-defrag-max-scan-fields) quicklist nodes (1000 ziplists, not 1000 items) by 16 bytes. In 32 bit build, this change reduces the maximum effective config of list-compress-depth and list-max-ziplist-size (from 32767 to 8191) --- src/defrag.c | 96 +++++++++++++++------- src/quicklist.c | 150 ++++++++++++++++++++++++++++++++++- src/quicklist.h | 46 ++++++++++- tests/unit/memefficiency.tcl | 92 +++++++++++++++++++++ 4 files changed, 350 insertions(+), 34 deletions(-) diff --git a/src/defrag.c b/src/defrag.c index 04e57955b..e729297a5 100644 --- a/src/defrag.c +++ b/src/defrag.c @@ -5,8 +5,8 @@ * We do that by scanning the keyspace and for each pointer we have, we can try to * ask the allocator if moving it to a new address will help reduce fragmentation. * - * Copyright (c) 2017, Oran Agra - * Copyright (c) 2017, Redis Labs, Inc + * Copyright (c) 2020, Oran Agra + * Copyright (c) 2020, Redis Labs, Inc * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -408,25 +408,32 @@ dictEntry* replaceSateliteDictKeyPtrAndOrDefragDictEntry(dict *d, sds oldkey, sd return NULL; } -long activeDefragQuickListNodes(quicklist *ql) { - quicklistNode *node = ql->head, *newnode; +long activeDefragQuickListNode(quicklist *ql, quicklistNode **node_ref) { + quicklistNode *newnode, *node = *node_ref; long defragged = 0; unsigned char *newzl; + if ((newnode = activeDefragAlloc(node))) { + if (newnode->prev) + newnode->prev->next = newnode; + else + ql->head = newnode; + if (newnode->next) + newnode->next->prev = newnode; + else + ql->tail = newnode; + *node_ref = node = newnode; + defragged++; + } + if ((newzl = activeDefragAlloc(node->zl))) + defragged++, node->zl = newzl; + return defragged; +} + +long activeDefragQuickListNodes(quicklist *ql) { + quicklistNode *node = ql->head; + long defragged = 0; while (node) { - if ((newnode = activeDefragAlloc(node))) { - if (newnode->prev) - newnode->prev->next = newnode; - else - ql->head = newnode; - if (newnode->next) - newnode->next->prev = newnode; - else - ql->tail = newnode; - node = newnode; - defragged++; - } - if ((newzl = activeDefragAlloc(node->zl))) - defragged++, node->zl = newzl; + defragged += activeDefragQuickListNode(ql, &node); node = node->next; } return defragged; @@ -440,12 +447,48 @@ void defragLater(redisDb *db, dictEntry *kde) { listAddNodeTail(db->defrag_later, key); } -long scanLaterList(robj *ob) { +/* returns 0 if no more work needs to be been done, and 1 if time is up and more work is needed. */ +long scanLaterList(robj *ob, unsigned long *cursor, long long endtime, long long *defragged) { quicklist *ql = ob->ptr; + quicklistNode *node; + long iterations = 0; + int bookmark_failed = 0; if (ob->type != OBJ_LIST || ob->encoding != OBJ_ENCODING_QUICKLIST) return 0; - server.stat_active_defrag_scanned+=ql->len; - return activeDefragQuickListNodes(ql); + + if (*cursor == 0) { + /* if cursor is 0, we start new iteration */ + node = ql->head; + } else { + node = quicklistBookmarkFind(ql, "_AD"); + if (!node) { + /* if the bookmark was deleted, it means we reached the end. */ + *cursor = 0; + return 0; + } + node = node->next; + } + + (*cursor)++; + while (node) { + (*defragged) += activeDefragQuickListNode(ql, &node); + server.stat_active_defrag_scanned++; + if (++iterations > 128 && !bookmark_failed) { + if (ustime() > endtime) { + if (!quicklistBookmarkCreate(&ql, "_AD", node)) { + bookmark_failed = 1; + } else { + ob->ptr = ql; /* bookmark creation may have re-allocated the quicklist */ + return 1; + } + } + iterations = 0; + } + node = node->next; + } + quicklistBookmarkDelete(ql, "_AD"); + *cursor = 0; + return bookmark_failed? 1: 0; } typedef struct { @@ -638,7 +681,8 @@ int scanLaterStraemListpacks(robj *ob, unsigned long *cursor, long long endtime, void *newdata = activeDefragAlloc(ri.data); if (newdata) raxSetData(ri.node, ri.data=newdata), (*defragged)++; - if (++iterations > 16) { + server.stat_active_defrag_scanned++; + if (++iterations > 128) { if (ustime() > endtime) { serverAssert(ri.key_len==sizeof(last)); memcpy(last,ri.key,ri.key_len); @@ -900,8 +944,7 @@ int defragLaterItem(dictEntry *de, unsigned long *cursor, long long endtime) { if (de) { robj *ob = dictGetVal(de); if (ob->type == OBJ_LIST) { - server.stat_active_defrag_hits += scanLaterList(ob); - *cursor = 0; /* list has no scan, we must finish it in one go */ + return scanLaterList(ob, cursor, endtime, &server.stat_active_defrag_hits); } else if (ob->type == OBJ_SET) { server.stat_active_defrag_hits += scanLaterSet(ob, cursor); } else if (ob->type == OBJ_ZSET) { @@ -961,11 +1004,6 @@ int defragLaterStep(redisDb *db, long long endtime) { if (defragLaterItem(de, &defrag_later_cursor, endtime)) quit = 1; /* time is up, we didn't finish all the work */ - /* Don't start a new BIG key in this loop, this is because the - * next key can be a list, and scanLaterList must be done in once cycle */ - if (!defrag_later_cursor) - quit = 1; - /* Once in 16 scan iterations, 512 pointer reallocations, or 64 fields * (if we have a lot of pointers in one hash bucket, or rehashing), * check if we reached the time limit. */ diff --git a/src/quicklist.c b/src/quicklist.c index 7b5484116..ae183ffd8 100644 --- a/src/quicklist.c +++ b/src/quicklist.c @@ -70,6 +70,12 @@ static const size_t optimization_level[] = {4096, 8192, 16384, 32768, 65536}; } while (0); #endif +/* Bookmarks forward declarations */ +#define QL_MAX_BM ((1 << QL_BM_BITS)-1) +quicklistBookmark *_quicklistBookmarkFindByName(quicklist *ql, const char *name); +quicklistBookmark *_quicklistBookmarkFindByNode(quicklist *ql, quicklistNode *node); +void _quicklistBookmarkDelete(quicklist *ql, quicklistBookmark *bm); + /* Simple way to give quicklistEntry structs default values with one call. */ #define initEntry(e) \ do { \ @@ -100,10 +106,11 @@ quicklist *quicklistCreate(void) { quicklist->count = 0; quicklist->compress = 0; quicklist->fill = -2; + quicklist->bookmark_count = 0; return quicklist; } -#define COMPRESS_MAX (1 << 16) +#define COMPRESS_MAX (1 << QL_COMP_BITS) void quicklistSetCompressDepth(quicklist *quicklist, int compress) { if (compress > COMPRESS_MAX) { compress = COMPRESS_MAX; @@ -113,7 +120,7 @@ void quicklistSetCompressDepth(quicklist *quicklist, int compress) { quicklist->compress = compress; } -#define FILL_MAX (1 << 15) +#define FILL_MAX (1 << (QL_FILL_BITS-1)) void quicklistSetFill(quicklist *quicklist, int fill) { if (fill > FILL_MAX) { fill = FILL_MAX; @@ -169,6 +176,7 @@ void quicklistRelease(quicklist *quicklist) { quicklist->len--; current = next; } + quicklistBookmarksClear(quicklist); zfree(quicklist); } @@ -578,6 +586,15 @@ quicklist *quicklistCreateFromZiplist(int fill, int compress, REDIS_STATIC void __quicklistDelNode(quicklist *quicklist, quicklistNode *node) { + /* Update the bookmark if any */ + quicklistBookmark *bm = _quicklistBookmarkFindByNode(quicklist, node); + if (bm) { + bm->node = node->next; + /* if the bookmark was to the last node, delete it. */ + if (!bm->node) + _quicklistBookmarkDelete(quicklist, bm); + } + if (node->next) node->next->prev = node->prev; if (node->prev) @@ -1410,6 +1427,87 @@ void quicklistPush(quicklist *quicklist, void *value, const size_t sz, } } +/* Create or update a bookmark in the list which will be updated to the next node + * automatically when the one referenced gets deleted. + * Returns 1 on success (creation of new bookmark or override of an existing one). + * Returns 0 on failure (reached the maximum supported number of bookmarks). + * NOTE: use short simple names, so that string compare on find is quick. + * NOTE: bookmakrk creation may re-allocate the quicklist, so the input pointer + may change and it's the caller responsibilty to update the reference. + */ +int quicklistBookmarkCreate(quicklist **ql_ref, const char *name, quicklistNode *node) { + quicklist *ql = *ql_ref; + if (ql->bookmark_count >= QL_MAX_BM) + return 0; + quicklistBookmark *bm = _quicklistBookmarkFindByName(ql, name); + if (bm) { + bm->node = node; + return 1; + } + ql = zrealloc(ql, sizeof(quicklist) + (ql->bookmark_count+1) * sizeof(quicklistBookmark)); + *ql_ref = ql; + ql->bookmarks[ql->bookmark_count].node = node; + ql->bookmarks[ql->bookmark_count].name = zstrdup(name); + ql->bookmark_count++; + return 1; +} + +/* Find the quicklist node referenced by a named bookmark. + * When the bookmarked node is deleted the bookmark is updated to the next node, + * and if that's the last node, the bookmark is deleted (so find returns NULL). */ +quicklistNode *quicklistBookmarkFind(quicklist *ql, const char *name) { + quicklistBookmark *bm = _quicklistBookmarkFindByName(ql, name); + if (!bm) return NULL; + return bm->node; +} + +/* Delete a named bookmark. + * returns 0 if bookmark was not found, and 1 if deleted. + * Note that the bookmark memory is not freed yet, and is kept for future use. */ +int quicklistBookmarkDelete(quicklist *ql, const char *name) { + quicklistBookmark *bm = _quicklistBookmarkFindByName(ql, name); + if (!bm) + return 0; + _quicklistBookmarkDelete(ql, bm); + return 1; +} + +quicklistBookmark *_quicklistBookmarkFindByName(quicklist *ql, const char *name) { + unsigned i; + for (i=0; ibookmark_count; i++) { + if (!strcmp(ql->bookmarks[i].name, name)) { + return &ql->bookmarks[i]; + } + } + return NULL; +} + +quicklistBookmark *_quicklistBookmarkFindByNode(quicklist *ql, quicklistNode *node) { + unsigned i; + for (i=0; ibookmark_count; i++) { + if (ql->bookmarks[i].node == node) { + return &ql->bookmarks[i]; + } + } + return NULL; +} + +void _quicklistBookmarkDelete(quicklist *ql, quicklistBookmark *bm) { + int index = bm - ql->bookmarks; + zfree(bm->name); + ql->bookmark_count--; + memmove(bm, bm+1, (ql->bookmark_count - index)* sizeof(*bm)); + /* NOTE: We do not shrink (realloc) the quicklist yet (to avoid resonance, + * it may be re-used later (a call to realloc may NOP). */ +} + +void quicklistBookmarksClear(quicklist *ql) { + while (ql->bookmark_count) + zfree(ql->bookmarks[--ql->bookmark_count].name); + /* NOTE: We do not shrink (realloc) the quick list. main use case for this + * function is just before releasing the allocation. */ +} + /* The rest of this file is test cases and test helpers. */ #ifdef REDIS_TEST #include @@ -2641,6 +2739,54 @@ int quicklistTest(int argc, char *argv[]) { printf("Compressions: %0.2f seconds.\n", (float)(stop - start) / 1000); printf("\n"); + TEST("bookmark get updated to next item") { + quicklist *ql = quicklistNew(1, 0); + quicklistPushTail(ql, "1", 1); + quicklistPushTail(ql, "2", 1); + quicklistPushTail(ql, "3", 1); + quicklistPushTail(ql, "4", 1); + quicklistPushTail(ql, "5", 1); + assert(ql->len==5); + /* add two bookmarks, one pointing to the node before the last. */ + assert(quicklistBookmarkCreate(&ql, "_dummy", ql->head->next)); + assert(quicklistBookmarkCreate(&ql, "_test", ql->tail->prev)); + /* test that the bookmark returns the right node, delete it and see that the bookmark points to the last node */ + assert(quicklistBookmarkFind(ql, "_test") == ql->tail->prev); + assert(quicklistDelRange(ql, -2, 1)); + assert(quicklistBookmarkFind(ql, "_test") == ql->tail); + /* delete the last node, and see that the bookmark was deleted. */ + assert(quicklistDelRange(ql, -1, 1)); + assert(quicklistBookmarkFind(ql, "_test") == NULL); + /* test that other bookmarks aren't affected */ + assert(quicklistBookmarkFind(ql, "_dummy") == ql->head->next); + assert(quicklistBookmarkFind(ql, "_missing") == NULL); + assert(ql->len==3); + quicklistBookmarksClear(ql); /* for coverage */ + assert(quicklistBookmarkFind(ql, "_dummy") == NULL); + quicklistRelease(ql); + } + + TEST("bookmark limit") { + int i; + quicklist *ql = quicklistNew(1, 0); + quicklistPushHead(ql, "1", 1); + for (i=0; ihead)); + /* when all bookmarks are used, creation fails */ + assert(!quicklistBookmarkCreate(&ql, "_test", ql->head)); + /* delete one and see that we can now create another */ + assert(quicklistBookmarkDelete(ql, "0")); + assert(quicklistBookmarkCreate(&ql, "_test", ql->head)); + /* delete one and see that the rest survive */ + assert(quicklistBookmarkDelete(ql, "_test")); + for (i=1; ihead); + /* make sure the deleted ones are indeed gone */ + assert(!quicklistBookmarkFind(ql, "0")); + assert(!quicklistBookmarkFind(ql, "_test")); + quicklistRelease(ql); + } + if (!err) printf("ALL TESTS PASSED!\n"); else diff --git a/src/quicklist.h b/src/quicklist.h index a7e27a2dd..8b553c119 100644 --- a/src/quicklist.h +++ b/src/quicklist.h @@ -28,6 +28,8 @@ * POSSIBILITY OF SUCH DAMAGE. */ +#include // for UINTPTR_MAX + #ifndef __QUICKLIST_H__ #define __QUICKLIST_H__ @@ -64,19 +66,51 @@ typedef struct quicklistLZF { char compressed[]; } quicklistLZF; +/* Bookmarks are padded with realloc at the end of of the quicklist struct. + * They should only be used for very big lists if thousands of nodes were the + * excess memory usage is negligible, and there's a real need to iterate on them + * in portions. + * When not used, they don't add any memory overhead, but when used and then + * deleted, some overhead remains (to avoid resonance). + * The number of bookmarks used should be kept to minimum since it also adds + * overhead on node deletion (searching for a bookmark to update). */ +typedef struct quicklistBookmark { + quicklistNode *node; + char *name; +} quicklistBookmark; + +#if UINTPTR_MAX == 0xffffffff +/* 32-bit */ +# define QL_FILL_BITS 14 +# define QL_COMP_BITS 14 +# define QL_BM_BITS 4 +#elif UINTPTR_MAX == 0xffffffffffffffff +/* 64-bit */ +# define QL_FILL_BITS 16 +# define QL_COMP_BITS 16 +# define QL_BM_BITS 4 /* we can encode more, but we rather limit the user + since they cause performance degradation. */ +#else +# error unknown arch bits count +#endif + /* quicklist is a 40 byte struct (on 64-bit systems) describing a quicklist. * 'count' is the number of total entries. * 'len' is the number of quicklist nodes. * 'compress' is: -1 if compression disabled, otherwise it's the number * of quicklistNodes to leave uncompressed at ends of quicklist. - * 'fill' is the user-requested (or default) fill factor. */ + * 'fill' is the user-requested (or default) fill factor. + * 'bookmakrs are an optional feature that is used by realloc this struct, + * so that they don't consume memory when not used. */ typedef struct quicklist { quicklistNode *head; quicklistNode *tail; unsigned long count; /* total count of all entries in all ziplists */ unsigned long len; /* number of quicklistNodes */ - int fill : 16; /* fill factor for individual nodes */ - unsigned int compress : 16; /* depth of end nodes not to compress;0=off */ + int fill : QL_FILL_BITS; /* fill factor for individual nodes */ + unsigned int compress : QL_COMP_BITS; /* depth of end nodes not to compress;0=off */ + unsigned int bookmark_count: QL_BM_BITS; + quicklistBookmark bookmarks[]; } quicklist; typedef struct quicklistIter { @@ -158,6 +192,12 @@ unsigned long quicklistCount(const quicklist *ql); int quicklistCompare(unsigned char *p1, unsigned char *p2, int p2_len); size_t quicklistGetLzf(const quicklistNode *node, void **data); +/* bookmarks */ +int quicklistBookmarkCreate(quicklist **ql_ref, const char *name, quicklistNode *node); +int quicklistBookmarkDelete(quicklist *ql, const char *name); +quicklistNode *quicklistBookmarkFind(quicklist *ql, const char *name); +void quicklistBookmarksClear(quicklist *ql); + #ifdef REDIS_TEST int quicklistTest(int argc, char *argv[]); #endif diff --git a/tests/unit/memefficiency.tcl b/tests/unit/memefficiency.tcl index d152e212c..ec80c7384 100644 --- a/tests/unit/memefficiency.tcl +++ b/tests/unit/memefficiency.tcl @@ -209,5 +209,97 @@ start_server {tags {"defrag"}} { assert {$digest eq $newdigest} r save ;# saving an rdb iterates over all the data / pointers } {OK} + + test "Active defrag big list" { + r flushdb + r config resetstat + r config set save "" ;# prevent bgsave from interfereing with save below + r config set hz 100 + r config set activedefrag no + r config set active-defrag-max-scan-fields 1000 + r config set active-defrag-threshold-lower 5 + r config set active-defrag-cycle-min 65 + r config set active-defrag-cycle-max 75 + r config set active-defrag-ignore-bytes 2mb + r config set maxmemory 0 + r config set list-max-ziplist-size 5 ;# list of 500k items will have 100k quicklist nodes + + # create big keys with 10k items + set rd [redis_deferring_client] + + set expected_frag 1.7 + # add a mass of list nodes to two lists (allocations are interlaced) + set val [string repeat A 100] ;# 5 items of 100 bytes puts us in the 640 bytes bin, which has 32 regs, so high potential for fragmentation + for {set j 0} {$j < 500000} {incr j} { + $rd lpush biglist1 $val + $rd lpush biglist2 $val + } + for {set j 0} {$j < 500000} {incr j} { + $rd read ; # Discard replies + $rd read ; # Discard replies + } + + # create some fragmentation + r del biglist2 + + # start defrag + after 120 ;# serverCron only updates the info once in 100ms + set frag [s allocator_frag_ratio] + if {$::verbose} { + puts "frag $frag" + } + + assert {$frag >= $expected_frag} + r config set latency-monitor-threshold 5 + r latency reset + + set digest [r debug digest] + catch {r config set activedefrag yes} e + if {![string match {DISABLED*} $e]} { + # wait for the active defrag to start working (decision once a second) + wait_for_condition 50 100 { + [s active_defrag_running] ne 0 + } else { + fail "defrag not started." + } + + # wait for the active defrag to stop working + wait_for_condition 500 100 { + [s active_defrag_running] eq 0 + } else { + after 120 ;# serverCron only updates the info once in 100ms + puts [r info memory] + puts [r info stats] + puts [r memory malloc-stats] + fail "defrag didn't stop." + } + + # test the the fragmentation is lower + after 120 ;# serverCron only updates the info once in 100ms + set frag [s allocator_frag_ratio] + set max_latency 0 + foreach event [r latency latest] { + lassign $event eventname time latency max + if {$eventname == "active-defrag-cycle"} { + set max_latency $max + } + } + if {$::verbose} { + puts "frag $frag" + puts "max latency $max_latency" + puts [r latency latest] + puts [r latency history active-defrag-cycle] + } + assert {$frag < 1.1} + # due to high fragmentation, 100hz, and active-defrag-cycle-max set to 75, + # we expect max latency to be not much higher than 7.5ms + assert {$max_latency <= 12} + } + # verify the data isn't corrupted or changed + set newdigest [r debug digest] + assert {$digest eq $newdigest} + r save ;# saving an rdb iterates over all the data / pointers + r del biglist1 ;# coverage for quicklistBookmarksClear + } {1} } } From 72c053519c282a8fea2a6c10578ca9dacc183cac Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 21 Feb 2020 18:55:56 +0100 Subject: [PATCH 083/225] Test engine: detect timeout when checking for Redis startup. --- tests/support/server.tcl | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/tests/support/server.tcl b/tests/support/server.tcl index 174b05852..eec43e485 100644 --- a/tests/support/server.tcl +++ b/tests/support/server.tcl @@ -271,9 +271,19 @@ proc start_server {options {code undefined}} { } # Wait for actual startup + set checkperiod 100; # Milliseconds + set maxiter [expr {120*1000/100}] ; # Wait up to 2 minutes. while {![info exists _pid]} { regexp {PID:\s(\d+)} [exec cat $stdout] _ _pid - after 100 + after $checkperiod + incr maxiter -1 + if {$maxiter == 0} { + start_server_error $config_file "No PID detected in log $stdout" + puts "--- LOG CONTENT ---" + puts [exec cat $stdout] + puts "-------------------" + break + } } # setup properties to be able to initialize a client object From 1bb5ee9c683835c6bb542c31ccb5b3e11f006900 Mon Sep 17 00:00:00 2001 From: hwware Date: Thu, 16 Jan 2020 17:33:23 -0500 Subject: [PATCH 084/225] fix potentical memory leaks --- src/debug.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/debug.c b/src/debug.c index dd96ad416..77d3e97dc 100644 --- a/src/debug.c +++ b/src/debug.c @@ -685,9 +685,12 @@ NULL sds stats = sdsempty(); char buf[4096]; - if (getLongFromObjectOrReply(c, c->argv[2], &dbid, NULL) != C_OK) + if (getLongFromObjectOrReply(c, c->argv[2], &dbid, NULL) != C_OK){ + sdsfree(stats); return; + } if (dbid < 0 || dbid >= server.dbnum) { + sdsfree(stats); addReplyError(c,"Out of range database"); return; } From ef3551d1499f5ea4bfc75d33c6ff1979b0aa9b07 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 24 Feb 2020 10:46:23 +0100 Subject: [PATCH 085/225] Test engine: experimental change to avoid busy port problems. --- tests/support/server.tcl | 149 ++++++++++++++++++++++++--------------- 1 file changed, 92 insertions(+), 57 deletions(-) diff --git a/tests/support/server.tcl b/tests/support/server.tcl index eec43e485..d086366dc 100644 --- a/tests/support/server.tcl +++ b/tests/support/server.tcl @@ -141,6 +141,18 @@ proc tags {tags code} { uplevel 1 $code set ::tags [lrange $::tags 0 end-[llength $tags]] } + +# Write the configuration in the dictionary 'config' in the specified +# file name. +proc create_server_config_file {filename config} { + set fp [open $filename w+] + foreach directive [dict keys $config] { + puts -nonewline $fp "$directive " + puts $fp [dict get $config $directive] + } + close $fp +} + proc start_server {options {code undefined}} { # If we are running against an external server, we just push the # host/port pair in the stack the first time @@ -222,68 +234,91 @@ proc start_server {options {code undefined}} { # write new configuration to temporary file set config_file [tmpfile redis.conf] - set fp [open $config_file w+] - foreach directive [dict keys $config] { - puts -nonewline $fp "$directive " - puts $fp [dict get $config $directive] - } - close $fp + create_server_config_file $config_file $config set stdout [format "%s/%s" [dict get $config "dir"] "stdout"] set stderr [format "%s/%s" [dict get $config "dir"] "stderr"] - send_data_packet $::test_server_fd "server-spawning" "port $::port" - - if {$::valgrind} { - set pid [exec valgrind --track-origins=yes --suppressions=src/valgrind.sup --show-reachable=no --show-possibly-lost=no --leak-check=full src/redis-server $config_file > $stdout 2> $stderr &] - } elseif ($::stack_logging) { - set pid [exec /usr/bin/env MallocStackLogging=1 MallocLogFile=/tmp/malloc_log.txt src/redis-server $config_file > $stdout 2> $stderr &] - } else { - set pid [exec src/redis-server $config_file > $stdout 2> $stderr &] - } - - # Tell the test server about this new instance. - send_data_packet $::test_server_fd server-spawned $pid - - # check that the server actually started - # ugly but tries to be as fast as possible... - if {$::valgrind} {set retrynum 1000} else {set retrynum 100} - - if {$::verbose} { - puts -nonewline "=== ($tags) Starting server ${::host}:${::port} " - } - - if {$code ne "undefined"} { - set serverisup [server_is_up $::host $::port $retrynum] - } else { - set serverisup 1 - } - - if {$::verbose} { - puts "" - } - - if {!$serverisup} { - set err {} - append err [exec cat $stdout] "\n" [exec cat $stderr] - start_server_error $config_file $err - return - } - - # Wait for actual startup - set checkperiod 100; # Milliseconds - set maxiter [expr {120*1000/100}] ; # Wait up to 2 minutes. - while {![info exists _pid]} { - regexp {PID:\s(\d+)} [exec cat $stdout] _ _pid - after $checkperiod - incr maxiter -1 - if {$maxiter == 0} { - start_server_error $config_file "No PID detected in log $stdout" - puts "--- LOG CONTENT ---" - puts [exec cat $stdout] - puts "-------------------" - break + # We need a loop here to retry with different ports. + set server_started 0 + while {$server_started == 0} { + if {$::verbose} { + puts -nonewline "=== ($tags) Starting server ${::host}:${::port} " } + + send_data_packet $::test_server_fd "server-spawning" "port $::port" + + if {$::valgrind} { + set pid [exec valgrind --track-origins=yes --suppressions=src/valgrind.sup --show-reachable=no --show-possibly-lost=no --leak-check=full src/redis-server $config_file > $stdout 2> $stderr &] + } elseif ($::stack_logging) { + set pid [exec /usr/bin/env MallocStackLogging=1 MallocLogFile=/tmp/malloc_log.txt src/redis-server $config_file > $stdout 2> $stderr &] + } else { + set pid [exec src/redis-server $config_file > $stdout 2> $stderr &] + } + + # Tell the test server about this new instance. + send_data_packet $::test_server_fd server-spawned $pid + + # check that the server actually started + # ugly but tries to be as fast as possible... + if {$::valgrind} {set retrynum 1000} else {set retrynum 100} + + # Wait for actual startup + set checkperiod 100; # Milliseconds + set maxiter [expr {120*1000/100}] ; # Wait up to 2 minutes. + set port_busy 0 + while {![info exists _pid]} { + regexp {PID:\s(\d+)} [exec cat $stdout] _ _pid + after $checkperiod + incr maxiter -1 + if {$maxiter == 0} { + start_server_error $config_file "No PID detected in log $stdout" + puts "--- LOG CONTENT ---" + puts [exec cat $stdout] + puts "-------------------" + break + } + + # Check if the port is actually busy and the server failed + # for this reason. + if {[regexp {Could not create server TCP} [exec cat $stdout]]} { + set port_busy 1 + break + } + } + + # Sometimes we have to try a different port, even if we checked + # for availability. Other test clients may grab the port before we + # are able to do it for example. + if {$port_busy} { + puts "Port $::port was already busy, trying another port..." + set ::port [find_available_port [expr {$::port+1}]] + if {$::tls} { + dict set config "tls-port" $::port + } else { + dict set config port $::port + } + create_server_config_file $config_file $config + continue; # Try again + } + + if {$code ne "undefined"} { + set serverisup [server_is_up $::host $::port $retrynum] + } else { + set serverisup 1 + } + + if {$::verbose} { + puts "" + } + + if {!$serverisup} { + set err {} + append err [exec cat $stdout] "\n" [exec cat $stderr] + start_server_error $config_file $err + return + } + set server_started 1 } # setup properties to be able to initialize a client object From 7277e5d8a83cb4d8eed94684c26f83c783d9838f Mon Sep 17 00:00:00 2001 From: hwware Date: Thu, 16 Jan 2020 17:35:26 -0500 Subject: [PATCH 086/225] format fix --- src/debug.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/debug.c b/src/debug.c index 77d3e97dc..36af35aec 100644 --- a/src/debug.c +++ b/src/debug.c @@ -685,7 +685,7 @@ NULL sds stats = sdsempty(); char buf[4096]; - if (getLongFromObjectOrReply(c, c->argv[2], &dbid, NULL) != C_OK){ + if (getLongFromObjectOrReply(c, c->argv[2], &dbid, NULL) != C_OK) { sdsfree(stats); return; } From b43954260452fb600d9cc5c30aa96c17804bc1b5 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 21 Feb 2020 16:39:42 +0100 Subject: [PATCH 087/225] Tracking: optin/out implemented. --- src/networking.c | 65 ++++++++++++++++++++++++++++++++++++++++++------ src/server.h | 6 ++++- src/tracking.c | 27 ++++++++++++++------ 3 files changed, 82 insertions(+), 16 deletions(-) diff --git a/src/networking.c b/src/networking.c index 5b1229fde..4c394af70 100644 --- a/src/networking.c +++ b/src/networking.c @@ -1365,6 +1365,12 @@ void resetClient(client *c) { if (!(c->flags & CLIENT_MULTI) && prevcmd != askingCommand) c->flags &= ~CLIENT_ASKING; + /* We do the same for the CACHING command as well. It also affects + * the next command or transaction executed, in a way very similar + * to ASKING. */ + if (!(c->flags & CLIENT_MULTI) && prevcmd != clientCommand) + c->flags &= ~CLIENT_TRACKING_CACHING; + /* Remove the CLIENT_REPLY_SKIP flag if any so that the reply * to the next command will be sent, but set the flag if the command * we just processed was "CLIENT REPLY SKIP". */ @@ -2044,7 +2050,7 @@ void clientCommand(client *c) { "REPLY (on|off|skip) -- Control the replies sent to the current connection.", "SETNAME -- Assign the name to the current connection.", "UNBLOCK [TIMEOUT|ERROR] -- Unblock the specified blocked client.", -"TRACKING (on|off) [REDIRECT ] [BCAST] [PREFIX first] [PREFIX second] ... -- Enable client keys tracking for client side caching.", +"TRACKING (on|off) [REDIRECT ] [BCAST] [PREFIX first] [PREFIX second] [OPTIN] [OPTOUT]... -- Enable client keys tracking for client side caching.", "GETREDIR -- Return the client ID we are redirecting to when tracking is enabled.", NULL }; @@ -2221,9 +2227,9 @@ NULL addReply(c,shared.ok); } else if (!strcasecmp(c->argv[1]->ptr,"tracking") && c->argc >= 3) { /* CLIENT TRACKING (on|off) [REDIRECT ] [BCAST] [PREFIX first] - * [PREFIX second] ... */ + * [PREFIX second] [OPTIN] [OPTOUT] ... */ long long redir = 0; - int bcast = 0; + uint64_t options = 0; robj **prefix = NULL; size_t numprefix = 0; @@ -2256,7 +2262,11 @@ NULL return; } } else if (!strcasecmp(c->argv[j]->ptr,"bcast")) { - bcast = 1; + options |= CLIENT_TRACKING_BCAST; + } else if (!strcasecmp(c->argv[j]->ptr,"optin")) { + options |= CLIENT_TRACKING_OPTIN; + } else if (!strcasecmp(c->argv[j]->ptr,"optout")) { + options |= CLIENT_TRACKING_OPTOUT; } else if (!strcasecmp(c->argv[j]->ptr,"prefix") && moreargs) { j++; prefix = zrealloc(prefix,sizeof(robj*)*(numprefix+1)); @@ -2272,7 +2282,7 @@ NULL if (!strcasecmp(c->argv[2]->ptr,"on")) { /* Before enabling tracking, make sure options are compatible * among each other and with the current state of the client. */ - if (!bcast && numprefix) { + if (!(options & CLIENT_TRACKING_BCAST) && numprefix) { addReplyError(c, "PREFIX option requires BCAST mode to be enabled"); zfree(prefix); @@ -2281,7 +2291,8 @@ NULL if (c->flags & CLIENT_TRACKING) { int oldbcast = !!(c->flags & CLIENT_TRACKING_BCAST); - if (oldbcast != bcast) { + int newbcast = !!(options & CLIENT_TRACKING_BCAST); + if (oldbcast != newbcast) { addReplyError(c, "You can't switch BCAST mode on/off before disabling " "tracking for this client, and then re-enabling it with " @@ -2290,7 +2301,17 @@ NULL return; } } - enableTracking(c,redir,bcast,prefix,numprefix); + + if (options & CLIENT_TRACKING_BCAST && + options & (CLIENT_TRACKING_OPTIN|CLIENT_TRACKING_OPTOUT)) + { + addReplyError(c, + "OPTIN and OPTOUT are not compatible with BCAST"); + zfree(prefix); + return; + } + + enableTracking(c,redir,options,prefix,numprefix); } else if (!strcasecmp(c->argv[2]->ptr,"off")) { disableTracking(c); } else { @@ -2300,6 +2321,36 @@ NULL } zfree(prefix); addReply(c,shared.ok); + } else if (!strcasecmp(c->argv[1]->ptr,"caching") && c->argc >= 3) { + if (!(c->flags & CLIENT_TRACKING)) { + addReplyError(c,"CLIENT CACHING can be called only when the " + "client is in tracking mode with OPTIN or " + "OPTOUT mode enabled"); + return; + } + + char *opt = c->argv[2]->ptr; + if (!strcasecmp(opt,"yes")) { + if (c->flags & CLIENT_TRACKING_OPTIN) { + c->flags |= CLIENT_TRACKING_CACHING; + } else { + addReplyError(c,"CLIENT CACHING YES is only valid when tracking is enabled in OPTIN mode."); + return; + } + } else if (!strcasecmp(opt,"no")) { + if (c->flags & CLIENT_TRACKING_OPTOUT) { + c->flags |= CLIENT_TRACKING_CACHING; + } else { + addReplyError(c,"CLIENT CACHING NO is only valid when tracking is enabled in OPTOUT mode."); + return; + } + } else { + addReply(c,shared.syntaxerr); + return; + } + + /* Common reply for when we succeeded. */ + addReply(c,shared.ok); } else if (!strcasecmp(c->argv[1]->ptr,"getredir") && c->argc == 2) { /* CLIENT GETREDIR */ if (c->flags & CLIENT_TRACKING) { diff --git a/src/server.h b/src/server.h index bb40ca4e1..87c293c26 100644 --- a/src/server.h +++ b/src/server.h @@ -248,6 +248,10 @@ typedef long long ustime_t; /* microsecond time type. */ perform client side caching. */ #define CLIENT_TRACKING_BROKEN_REDIR (1ULL<<32) /* Target client is invalid. */ #define CLIENT_TRACKING_BCAST (1ULL<<33) /* Tracking in BCAST mode. */ +#define CLIENT_TRACKING_OPTIN (1ULL<<34) /* Tracking in opt-in mode. */ +#define CLIENT_TRACKING_OPTOUT (1ULL<<35) /* Tracking in opt-out mode. */ +#define CLIENT_TRACKING_CACHING (1ULL<<36) /* CACHING yes/no was given, + depending on optin/optout mode. */ /* Client block type (btype field in client structure) * if CLIENT_BLOCKED flag is set. */ @@ -1651,7 +1655,7 @@ void addReplyStatusFormat(client *c, const char *fmt, ...); #endif /* Client side caching (tracking mode) */ -void enableTracking(client *c, uint64_t redirect_to, int bcast, robj **prefix, size_t numprefix); +void enableTracking(client *c, uint64_t redirect_to, uint64_t options, robj **prefix, size_t numprefix); void disableTracking(client *c); void trackingRememberKeys(client *c); void trackingInvalidateKey(robj *keyobj); diff --git a/src/tracking.c b/src/tracking.c index 619148f2f..45f83103a 100644 --- a/src/tracking.c +++ b/src/tracking.c @@ -93,7 +93,8 @@ void disableTracking(client *c) { if (c->flags & CLIENT_TRACKING) { server.tracking_clients--; c->flags &= ~(CLIENT_TRACKING|CLIENT_TRACKING_BROKEN_REDIR| - CLIENT_TRACKING_BCAST); + CLIENT_TRACKING_BCAST|CLIENT_TRACKING_OPTIN| + CLIENT_TRACKING_OPTOUT); } } @@ -124,10 +125,11 @@ void enableBcastTrackingForPrefix(client *c, char *prefix, size_t plen) { * eventually get freed, we'll send a message to the original client to * inform it of the condition. Multiple clients can redirect the invalidation * messages to the same client ID. */ -void enableTracking(client *c, uint64_t redirect_to, int bcast, robj **prefix, size_t numprefix) { +void enableTracking(client *c, uint64_t redirect_to, uint64_t options, robj **prefix, size_t numprefix) { if (!(c->flags & CLIENT_TRACKING)) server.tracking_clients++; c->flags |= CLIENT_TRACKING; - c->flags &= ~(CLIENT_TRACKING_BROKEN_REDIR|CLIENT_TRACKING_BCAST); + c->flags &= ~(CLIENT_TRACKING_BROKEN_REDIR|CLIENT_TRACKING_BCAST| + CLIENT_TRACKING_OPTIN|CLIENT_TRACKING_OPTOUT); c->client_tracking_redirection = redirect_to; if (TrackingTable == NULL) { TrackingTable = raxNew(); @@ -135,7 +137,7 @@ void enableTracking(client *c, uint64_t redirect_to, int bcast, robj **prefix, s TrackingChannelName = createStringObject("__redis__:invalidate",20); } - if (bcast) { + if (options & CLIENT_TRACKING_BCAST) { c->flags |= CLIENT_TRACKING_BCAST; if (numprefix == 0) enableBcastTrackingForPrefix(c,"",0); for (size_t j = 0; j < numprefix; j++) { @@ -143,14 +145,23 @@ void enableTracking(client *c, uint64_t redirect_to, int bcast, robj **prefix, s enableBcastTrackingForPrefix(c,sdsprefix,sdslen(sdsprefix)); } } + c->flags |= options & (CLIENT_TRACKING_OPTIN|CLIENT_TRACKING_OPTOUT); } /* This function is called after the execution of a readonly command in the - * case the client 'c' has keys tracking enabled. It will populate the - * tracking invalidation table according to the keys the user fetched, so that - * Redis will know what are the clients that should receive an invalidation - * message with certain groups of keys are modified. */ + * case the client 'c' has keys tracking enabled and the tracking is not + * in BCAST mode. It will populate the tracking invalidation table according + * to the keys the user fetched, so that Redis will know what are the clients + * that should receive an invalidation message with certain groups of keys + * are modified. */ void trackingRememberKeys(client *c) { + /* Return if we are in optin/out mode and the right CACHING command + * was/wasn't given in order to modify the default behavior. */ + uint64_t optin = c->flags & CLIENT_TRACKING_OPTIN; + uint64_t optout = c->flags & CLIENT_TRACKING_OPTOUT; + uint64_t caching_given = c->flags & CLIENT_TRACKING_CACHING; + if ((optin && !caching_given) || (optout && caching_given)) return; + int numkeys; int *keys = getKeysFromCommand(c->cmd,c->argv,c->argc,&numkeys); if (keys == NULL) return; From 2966132c520ffd2fc25472f7a202e37a64766639 Mon Sep 17 00:00:00 2001 From: "meir@redislabs.com" Date: Mon, 10 Feb 2020 12:10:32 +0200 Subject: [PATCH 088/225] Changed log level for module fork api from 'notice' to 'verbos'. --- src/module.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/module.c b/src/module.c index 8c61f60e2..cce3d1c11 100644 --- a/src/module.c +++ b/src/module.c @@ -6724,7 +6724,7 @@ int RM_Fork(RedisModuleForkDoneHandler cb, void *user_data) { server.module_child_pid = childpid; moduleForkInfo.done_handler = cb; moduleForkInfo.done_handler_user_data = user_data; - serverLog(LL_NOTICE, "Module fork started pid: %d ", childpid); + serverLog(LL_VERBOSE, "Module fork started pid: %d ", childpid); } return childpid; } @@ -6747,7 +6747,7 @@ int TerminateModuleForkChild(int child_pid, int wait) { server.module_child_pid != child_pid) return C_ERR; int statloc; - serverLog(LL_NOTICE,"Killing running module fork child: %ld", + serverLog(LL_VERBOSE,"Killing running module fork child: %ld", (long) server.module_child_pid); if (kill(server.module_child_pid,SIGUSR1) != -1 && wait) { while(wait4(server.module_child_pid,&statloc,0,NULL) != From 60096bc1a1d24e87a4ab9b4bcc671e1a9925476a Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Sun, 23 Feb 2020 12:46:14 +0200 Subject: [PATCH 089/225] Fix latency sensitivity of new defrag test I saw that the new defag test for list was failing in CI recently, so i reduce it's threshold from 12 to 60. besides that, i add / improve the latency test for that other two defrag tests (add a sensitive latency and digest / save checks) and fix bad usage of debug populate (can't overrides existing keys). this was the original intention, which creates higher fragmentation. --- tests/unit/memefficiency.tcl | 40 ++++++++++++++++++++++++++++-------- 1 file changed, 32 insertions(+), 8 deletions(-) diff --git a/tests/unit/memefficiency.tcl b/tests/unit/memefficiency.tcl index ec80c7384..468825a47 100644 --- a/tests/unit/memefficiency.tcl +++ b/tests/unit/memefficiency.tcl @@ -39,6 +39,8 @@ start_server {tags {"memefficiency"}} { start_server {tags {"defrag"}} { if {[string match {*jemalloc*} [s mem_allocator]]} { test "Active defrag" { + r config set save "" ;# prevent bgsave from interfereing with save below + r config set hz 100 r config set activedefrag no r config set active-defrag-threshold-lower 5 r config set active-defrag-cycle-min 65 @@ -46,8 +48,8 @@ start_server {tags {"defrag"}} { r config set active-defrag-ignore-bytes 2mb r config set maxmemory 100mb r config set maxmemory-policy allkeys-lru - r debug populate 700000 asdf 150 - r debug populate 170000 asdf 300 + r debug populate 700000 asdf1 150 + r debug populate 170000 asdf2 300 r ping ;# trigger eviction following the previous population after 120 ;# serverCron only updates the info once in 100ms set frag [s allocator_frag_ratio] @@ -55,6 +57,10 @@ start_server {tags {"defrag"}} { puts "frag $frag" } assert {$frag >= 1.4} + + r config set latency-monitor-threshold 5 + r latency reset + set digest [r debug digest] catch {r config set activedefrag yes} e if {![string match {DISABLED*} $e]} { # Wait for the active defrag to start working (decision once a @@ -78,19 +84,37 @@ start_server {tags {"defrag"}} { # Test the the fragmentation is lower. after 120 ;# serverCron only updates the info once in 100ms set frag [s allocator_frag_ratio] + set max_latency 0 + foreach event [r latency latest] { + lassign $event eventname time latency max + if {$eventname == "active-defrag-cycle"} { + set max_latency $max + } + } if {$::verbose} { puts "frag $frag" + puts "max latency $max_latency" + puts [r latency latest] + puts [r latency history active-defrag-cycle] } assert {$frag < 1.1} + # due to high fragmentation, 100hz, and active-defrag-cycle-max set to 75, + # we expect max latency to be not much higher than 7.5ms but due to rare slowness threshold is set higher + assert {$max_latency <= 60} } else { set _ "" } - } {} + # verify the data isn't corrupted or changed + set newdigest [r debug digest] + assert {$digest eq $newdigest} + r save ;# saving an rdb iterates over all the data / pointers + } {OK} test "Active defrag big keys" { r flushdb r config resetstat r config set save "" ;# prevent bgsave from interfereing with save below + r config set hz 100 r config set activedefrag no r config set active-defrag-max-scan-fields 1000 r config set active-defrag-threshold-lower 5 @@ -200,9 +224,9 @@ start_server {tags {"defrag"}} { puts [r latency history active-defrag-cycle] } assert {$frag < 1.1} - # due to high fragmentation, 10hz, and active-defrag-cycle-max set to 75, - # we expect max latency to be not much higher than 75ms - assert {$max_latency <= 120} + # due to high fragmentation, 100hz, and active-defrag-cycle-max set to 75, + # we expect max latency to be not much higher than 7.5ms but due to rare slowness threshold is set higher + assert {$max_latency <= 60} } # verify the data isn't corrupted or changed set newdigest [r debug digest] @@ -292,8 +316,8 @@ start_server {tags {"defrag"}} { } assert {$frag < 1.1} # due to high fragmentation, 100hz, and active-defrag-cycle-max set to 75, - # we expect max latency to be not much higher than 7.5ms - assert {$max_latency <= 12} + # we expect max latency to be not much higher than 7.5ms but due to rare slowness threshold is set higher + assert {$max_latency <= 60} } # verify the data isn't corrupted or changed set newdigest [r debug digest] From ecf3b2ef320f8e9d07ed05634be432d8bd9529db Mon Sep 17 00:00:00 2001 From: srzhao Date: Fri, 17 Jan 2020 11:46:19 +0800 Subject: [PATCH 090/225] fix impl of aof-child whitelist SIGUSR1 feature. --- src/aof.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/aof.c b/src/aof.c index 3682c4568..8ab9349f0 100644 --- a/src/aof.c +++ b/src/aof.c @@ -1798,14 +1798,15 @@ void backgroundRewriteDoneHandler(int exitcode, int bysignal) { serverLog(LL_VERBOSE, "Background AOF rewrite signal handler took %lldus", ustime()-now); } else if (!bysignal && exitcode != 0) { + server.aof_lastbgrewrite_status = C_ERR; + + serverLog(LL_WARNING, + "Background AOF rewrite terminated with error"); + } else { /* SIGUSR1 is whitelisted, so we have a way to kill a child without * tirggering an error condition. */ if (bysignal != SIGUSR1) server.aof_lastbgrewrite_status = C_ERR; - serverLog(LL_WARNING, - "Background AOF rewrite terminated with error"); - } else { - server.aof_lastbgrewrite_status = C_ERR; serverLog(LL_WARNING, "Background AOF rewrite terminated by signal %d", bysignal); From 0b988fa9ece7e92bd84d509ea281b03c120db74f Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Tue, 25 Feb 2020 13:01:52 +0200 Subject: [PATCH 091/225] fix github actions failing latency test for active defrag seems that github actions are slow, using just one client to reduce false positives. also adding verbose, testing only on latest ubuntu, and building on older one. when doing that, i can reduce the test threshold back to something saner --- .github/workflows/ci.yml | 23 ++++++++++++----------- tests/unit/memefficiency.tcl | 6 +++--- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 847abcf02..559ae61d8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3,11 +3,8 @@ name: CI on: [push, pull_request] jobs: - build-ubuntu: - strategy: - matrix: - platform: [ubuntu-latest, ubuntu-16.04] - runs-on: ${{ matrix.platform }} + test-ubuntu-latest: + runs-on: ubuntu-latest steps: - uses: actions/checkout@v1 - name: make @@ -15,13 +12,17 @@ jobs: - name: test run: | sudo apt-get install tcl8.5 - make test + ./runtest --clients 1 --verbose - build-macos-latest: - strategy: - matrix: - platform: [macos-latest, macOS-10.14] - runs-on: ${{ matrix.platform }} + test-ubuntu-old: + runs-on: ubuntu-16.04 + steps: + - uses: actions/checkout@v1 + - name: make + run: make + + build-macos-latest: + runs-on: macos-latest steps: - uses: actions/checkout@v1 - name: make diff --git a/tests/unit/memefficiency.tcl b/tests/unit/memefficiency.tcl index 468825a47..c899103fd 100644 --- a/tests/unit/memefficiency.tcl +++ b/tests/unit/memefficiency.tcl @@ -100,7 +100,7 @@ start_server {tags {"defrag"}} { assert {$frag < 1.1} # due to high fragmentation, 100hz, and active-defrag-cycle-max set to 75, # we expect max latency to be not much higher than 7.5ms but due to rare slowness threshold is set higher - assert {$max_latency <= 60} + assert {$max_latency <= 30} } else { set _ "" } @@ -226,7 +226,7 @@ start_server {tags {"defrag"}} { assert {$frag < 1.1} # due to high fragmentation, 100hz, and active-defrag-cycle-max set to 75, # we expect max latency to be not much higher than 7.5ms but due to rare slowness threshold is set higher - assert {$max_latency <= 60} + assert {$max_latency <= 30} } # verify the data isn't corrupted or changed set newdigest [r debug digest] @@ -317,7 +317,7 @@ start_server {tags {"defrag"}} { assert {$frag < 1.1} # due to high fragmentation, 100hz, and active-defrag-cycle-max set to 75, # we expect max latency to be not much higher than 7.5ms but due to rare slowness threshold is set higher - assert {$max_latency <= 60} + assert {$max_latency <= 30} } # verify the data isn't corrupted or changed set newdigest [r debug digest] From dd4798802c82ce674be00d908a66903f46dda5a6 Mon Sep 17 00:00:00 2001 From: wangyuan21 Date: Tue, 31 Dec 2019 19:53:00 +0800 Subject: [PATCH 092/225] free time event when delete eventloop --- src/ae.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/ae.c b/src/ae.c index 2c1dae512..248096e1f 100644 --- a/src/ae.c +++ b/src/ae.c @@ -135,6 +135,13 @@ void aeDeleteEventLoop(aeEventLoop *eventLoop) { aeApiFree(eventLoop); zfree(eventLoop->events); zfree(eventLoop->fired); + /* Free time event. */ + aeTimeEvent *next_te, *te = eventLoop->timeEventHead; + while (te) { + next_te = te->next; + zfree(te); + te = next_te; + } zfree(eventLoop); } From 635321d47e4a7fe70ab6a38a66a7b87d16455adb Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Wed, 26 Feb 2020 08:12:07 +0200 Subject: [PATCH 093/225] fix github actions failing latency test for active defrag - part 2 it seems that running two clients at a time is ok too, resuces action time from 20 minutes to 10. we'll use this for now, and if one day it won't be enough we'll have to run just the sensitive tests one by one separately from the others. this commit also fixes an issue with the defrag test that appears to be very rare. --- .github/workflows/ci.yml | 4 ++-- tests/unit/memefficiency.tcl | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 559ae61d8..cc4991606 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,9 +12,9 @@ jobs: - name: test run: | sudo apt-get install tcl8.5 - ./runtest --clients 1 --verbose + ./runtest --clients 2 --verbose - test-ubuntu-old: + build-ubuntu-old: runs-on: ubuntu-16.04 steps: - uses: actions/checkout@v1 diff --git a/tests/unit/memefficiency.tcl b/tests/unit/memefficiency.tcl index c899103fd..06b0e07d7 100644 --- a/tests/unit/memefficiency.tcl +++ b/tests/unit/memefficiency.tcl @@ -60,6 +60,7 @@ start_server {tags {"defrag"}} { r config set latency-monitor-threshold 5 r latency reset + r config set maxmemory 110mb ;# prevent further eviction (not to fail the digest test) set digest [r debug digest] catch {r config set activedefrag yes} e if {![string match {DISABLED*} $e]} { @@ -166,7 +167,7 @@ start_server {tags {"defrag"}} { for {set j 0} {$j < 500000} {incr j} { $rd read ; # Discard replies } - assert {[r dbsize] == 500010} + assert_equal [r dbsize] 500010 # create some fragmentation for {set j 0} {$j < 500000} {incr j 2} { @@ -175,7 +176,7 @@ start_server {tags {"defrag"}} { for {set j 0} {$j < 500000} {incr j 2} { $rd read ; # Discard replies } - assert {[r dbsize] == 250010} + assert_equal [r dbsize] 250010 # start defrag after 120 ;# serverCron only updates the info once in 100ms From ea697b6345830f24811a293c9689a65bc438015f Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 27 Feb 2020 17:41:48 +0100 Subject: [PATCH 094/225] Improve aeDeleteEventLoop() top comment grammar. --- src/ae.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/ae.c b/src/ae.c index 248096e1f..d2248fe5c 100644 --- a/src/ae.c +++ b/src/ae.c @@ -135,7 +135,8 @@ void aeDeleteEventLoop(aeEventLoop *eventLoop) { aeApiFree(eventLoop); zfree(eventLoop->events); zfree(eventLoop->fired); - /* Free time event. */ + + /* Free the time events list. */ aeTimeEvent *next_te, *te = eventLoop->timeEventHead; while (te) { next_te = te->next; From 2ecab0b63a632672f7f60a83b578a8aba398487e Mon Sep 17 00:00:00 2001 From: Guy Benoish Date: Wed, 19 Feb 2020 13:24:50 +0530 Subject: [PATCH 095/225] Modules: Do not auto-unblock clients if not blocked on keys --- src/module.c | 29 ++++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/src/module.c b/src/module.c index aae15d74b..f37d987e0 100644 --- a/src/module.c +++ b/src/module.c @@ -4290,12 +4290,15 @@ void unblockClientFromModule(client *c) { * We must call moduleUnblockClient in order to free privdata and * RedisModuleBlockedClient. * - * Note that clients implementing threads and working with private data, - * should make sure to stop the threads or protect the private data - * in some other way in the disconnection and timeout callback, because - * here we are going to free the private data associated with the - * blocked client. */ - if (!bc->unblocked) + * Note that we only do that for clients that are blocked on keys, for which + * the contract is that the module should not call RM_UnblockClient under + * normal circumstances. + * Clients implementing threads and working with private data should be + * aware that calling RM_UnblockClient for every blocked client is their + * responsibility, and if they fail to do so memory may leak. Ideally they + * should implement the disconnect and timeout callbacks and call + * RM_UnblockClient, but any other way is also acceptable. */ + if (bc->blocked_on_keys && !bc->unblocked) moduleUnblockClient(c); bc->client = NULL; @@ -4409,6 +4412,10 @@ int moduleTryServeClientBlockedOnKey(client *c, robj *key) { * * free_privdata: called in order to free the private data that is passed * by RedisModule_UnblockClient() call. + * + * Note: RedisModule_UnblockClient should be called for every blocked client, + * even if client was killed, timed-out or disconnected. Failing to do so + * will result in memory leaks. */ RedisModuleBlockedClient *RM_BlockClient(RedisModuleCtx *ctx, RedisModuleCmdFunc reply_callback, RedisModuleCmdFunc timeout_callback, void (*free_privdata)(RedisModuleCtx*,void*), long long timeout_ms) { return moduleBlockClient(ctx,reply_callback,timeout_callback,free_privdata,timeout_ms, NULL,0,NULL); @@ -4463,7 +4470,15 @@ RedisModuleBlockedClient *RM_BlockClient(RedisModuleCtx *ctx, RedisModuleCmdFunc * freed using the free_privdata callback provided by the user. * * However the reply callback will be able to access the argument vector of - * the command, so the private data is often not needed. */ + * the command, so the private data is often not needed. + * + * Note: Under normal circumstances RedisModule_UnblockClient should not be + * called for clients that are blocked on keys (Either the key will + * become ready or a timeout will occur). If for some reason you do want + * to call RedisModule_UnblockClient it is possible: Client will be + * handled as if it were timed-out (You must implement the timeout + * callback in that case). + */ RedisModuleBlockedClient *RM_BlockClientOnKeys(RedisModuleCtx *ctx, RedisModuleCmdFunc reply_callback, RedisModuleCmdFunc timeout_callback, void (*free_privdata)(RedisModuleCtx*,void*), long long timeout_ms, RedisModuleString **keys, int numkeys, void *privdata) { return moduleBlockClient(ctx,reply_callback,timeout_callback,free_privdata,timeout_ms, keys,numkeys,privdata); } From dafb94db7958114cd6c68d30c6e975579566df82 Mon Sep 17 00:00:00 2001 From: Ponnuvel Palaniyappan Date: Tue, 14 Jan 2020 08:10:39 +0000 Subject: [PATCH 096/225] Fix a potential overflow with strncpy --- src/config.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/config.c b/src/config.c index d55d1f8b5..5841ae7a0 100644 --- a/src/config.c +++ b/src/config.c @@ -108,12 +108,12 @@ clientBufferLimitsConfig clientBufferLimitsDefaults[CLIENT_TYPE_OBUF_COUNT] = { /* Generic config infrastructure function pointers * int is_valid_fn(val, err) * Return 1 when val is valid, and 0 when invalid. - * Optionslly set err to a static error string. + * Optionally set err to a static error string. * int update_fn(val, prev, err) * This function is called only for CONFIG SET command (not at config file parsing) * It is called after the actual config is applied, * Return 1 for success, and 0 for failure. - * Optionslly set err to a static error string. + * Optionally set err to a static error string. * On failure the config change will be reverted. */ @@ -729,7 +729,7 @@ void configSetCommand(client *c) { * config_set_memory_field(name,var) */ } config_set_memory_field( "client-query-buffer-limit",server.client_max_querybuf_len) { - /* Everyhing else is an error... */ + /* Everything else is an error... */ } config_set_else { addReplyErrorFormat(c,"Unsupported CONFIG parameter: %s", (char*)c->argv[2]->ptr); @@ -1673,9 +1673,9 @@ static int enumConfigSet(typeData data, sds value, int update, char **err) { enumerr[sdslen(enumerr) - 2] = '\0'; - /* Make sure we don't overrun the fixed buffer */ - enumerr[LOADBUF_SIZE - 1] = '\0'; strncpy(loadbuf, enumerr, LOADBUF_SIZE); + /* strncpy does not if null terminate if source string length is >= destination buffer. */ + loadbuf[LOADBUF_SIZE - 1] = '\0'; sdsfree(enumerr); *err = loadbuf; From 12626ce9bb58bbc9f32d7ae94e343a827bd81663 Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Thu, 20 Feb 2020 17:56:52 +0200 Subject: [PATCH 097/225] fix race in module api test for fork in some cases we were trying to kill the fork before it got created --- tests/modules/fork.c | 2 +- tests/unit/moduleapi/fork.tcl | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/modules/fork.c b/tests/modules/fork.c index 1a139ef1b..0443d9ef0 100644 --- a/tests/modules/fork.c +++ b/tests/modules/fork.c @@ -42,7 +42,7 @@ int fork_create(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) /* child */ RedisModule_Log(ctx, "notice", "fork child started"); - usleep(200000); + usleep(500000); RedisModule_Log(ctx, "notice", "fork child exiting"); RedisModule_ExitFromChild(code_to_exit_with); /* unreachable */ diff --git a/tests/unit/moduleapi/fork.tcl b/tests/unit/moduleapi/fork.tcl index f7d7e47d5..8535a3382 100644 --- a/tests/unit/moduleapi/fork.tcl +++ b/tests/unit/moduleapi/fork.tcl @@ -20,9 +20,8 @@ start_server {tags {"modules"}} { test {Module fork kill} { r fork.create 3 - after 20 + after 250 r fork.kill - after 100 assert {[count_log_message "fork child started"] eq "2"} assert {[count_log_message "Received SIGUSR1 in child"] eq "1"} From 84243064337a60779b4ec14868be30f2c3eab490 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 27 Feb 2020 17:45:48 +0100 Subject: [PATCH 098/225] Remove useless comment from enumConfigSet(). --- src/config.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/config.c b/src/config.c index 5841ae7a0..8d069f8db 100644 --- a/src/config.c +++ b/src/config.c @@ -1674,7 +1674,6 @@ static int enumConfigSet(typeData data, sds value, int update, char **err) { enumerr[sdslen(enumerr) - 2] = '\0'; strncpy(loadbuf, enumerr, LOADBUF_SIZE); - /* strncpy does not if null terminate if source string length is >= destination buffer. */ loadbuf[LOADBUF_SIZE - 1] = '\0'; sdsfree(enumerr); From 4d12c37c54d81a89ccbbea3c23f783477fb97c51 Mon Sep 17 00:00:00 2001 From: Guy Benoish Date: Sun, 23 Feb 2020 19:13:09 +0530 Subject: [PATCH 099/225] XREADGROUP should propagate XCALIM/SETID in MULTI/EXEC Use built-in alsoPropagate mechanism that wraps commands in MULTI/EXEC before sending them to replica/AOF --- src/t_stream.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/t_stream.c b/src/t_stream.c index 900fa3a17..e1efc6bca 100644 --- a/src/t_stream.c +++ b/src/t_stream.c @@ -848,7 +848,7 @@ void streamPropagateXCLAIM(client *c, robj *key, streamCG *group, robj *groupnam argv[11] = createStringObject("JUSTID",6); argv[12] = createStringObject("LASTID",6); argv[13] = createObjectFromStreamID(&group->last_id); - propagate(server.xclaimCommand,c->db->id,argv,14,PROPAGATE_AOF|PROPAGATE_REPL); + alsoPropagate(server.xclaimCommand,c->db->id,argv,14,PROPAGATE_AOF|PROPAGATE_REPL); decrRefCount(argv[0]); decrRefCount(argv[3]); decrRefCount(argv[4]); @@ -875,7 +875,7 @@ void streamPropagateGroupID(client *c, robj *key, streamCG *group, robj *groupna argv[2] = key; argv[3] = groupname; argv[4] = createObjectFromStreamID(&group->last_id); - propagate(server.xgroupCommand,c->db->id,argv,5,PROPAGATE_AOF|PROPAGATE_REPL); + alsoPropagate(server.xgroupCommand,c->db->id,argv,5,PROPAGATE_AOF|PROPAGATE_REPL); decrRefCount(argv[0]); decrRefCount(argv[1]); decrRefCount(argv[4]); From 9d4219ebac8fe36dc5f8504e1b3ed1dec30bfa84 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 27 Feb 2020 17:47:50 +0100 Subject: [PATCH 100/225] Fix SDS misuse in enumConfigSet(). Related to #6778. --- src/config.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/config.c b/src/config.c index 8d069f8db..aeb2fea7e 100644 --- a/src/config.c +++ b/src/config.c @@ -1666,12 +1666,12 @@ static int enumConfigSet(typeData data, sds value, int update, char **err) { sds enumerr = sdsnew("argument must be one of the following: "); configEnum *enumNode = data.enumd.enum_value; while(enumNode->name != NULL) { - enumerr = sdscatlen(enumerr, enumNode->name, strlen(enumNode->name)); + enumerr = sdscatlen(enumerr, enumNode->name, + strlen(enumNode->name)); enumerr = sdscatlen(enumerr, ", ", 2); enumNode++; } - - enumerr[sdslen(enumerr) - 2] = '\0'; + sdsrange(enumerr,0,-3); /* Remove final ", ". */ strncpy(loadbuf, enumerr, LOADBUF_SIZE); loadbuf[LOADBUF_SIZE - 1] = '\0'; From 15ea13245ae388a34277595efaf17e94cb6b6bc1 Mon Sep 17 00:00:00 2001 From: Ariel Date: Sat, 22 Feb 2020 23:49:23 +0200 Subject: [PATCH 101/225] fix ThreadSafeContext lock/unlock function names --- src/module.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/module.c b/src/module.c index f37d987e0..8c61f60e2 100644 --- a/src/module.c +++ b/src/module.c @@ -4740,9 +4740,9 @@ int RM_BlockedClientDisconnected(RedisModuleCtx *ctx) { * * To call non-reply APIs, the thread safe context must be prepared with: * - * RedisModule_ThreadSafeCallStart(ctx); + * RedisModule_ThreadSafeContextLock(ctx); * ... make your call here ... - * RedisModule_ThreadSafeCallStop(ctx); + * RedisModule_ThreadSafeContextUnlock(ctx); * * This is not needed when using `RedisModule_Reply*` functions, assuming * that a blocked client was used when the context was created, otherwise From fe902461f467c28de15803cff8455abddab1288b Mon Sep 17 00:00:00 2001 From: "bodong.ybd" Date: Sat, 21 Dec 2019 21:27:38 +0800 Subject: [PATCH 102/225] Fix spop return nil #4709 --- src/t_set.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/t_set.c b/src/t_set.c index abbf82fde..60cf22d8c 100644 --- a/src/t_set.c +++ b/src/t_set.c @@ -415,7 +415,7 @@ void spopWithCountCommand(client *c) { /* Make sure a key with the name inputted exists, and that it's type is * indeed a set. Otherwise, return nil */ - if ((set = lookupKeyWriteOrReply(c,c->argv[1],shared.null[c->resp])) + if ((set = lookupKeyWriteOrReply(c,c->argv[1],shared.emptyset[c->resp])) == NULL || checkType(c,set,OBJ_SET)) return; /* If count is zero, serve an empty set ASAP to avoid special From 973297336fc05a601e17be70aba88e5dca6480ae Mon Sep 17 00:00:00 2001 From: Hengjian Tang Date: Tue, 25 Feb 2020 15:55:28 +0800 Subject: [PATCH 103/225] modify the read buf size according to the write buf size PROTO_IOBUF_LEN defined before --- src/replication.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/replication.c b/src/replication.c index 4843f97d5..c497051c8 100644 --- a/src/replication.c +++ b/src/replication.c @@ -1352,7 +1352,7 @@ void disklessLoadRestoreBackups(redisDb *backup, int restore, int empty_db_flags /* Asynchronously read the SYNC payload we receive from a master */ #define REPL_MAX_WRITTEN_BEFORE_FSYNC (1024*1024*8) /* 8 MB */ void readSyncBulkPayload(connection *conn) { - char buf[4096]; + char buf[PROTO_IOBUF_LEN]; ssize_t nread, readlen, nwritten; int use_diskless_load = useDisklessLoad(); redisDb *diskless_load_backup = NULL; From 141c0679a03795fc395ef8b03818902c8a8fe24a Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 27 Feb 2020 18:04:34 +0100 Subject: [PATCH 104/225] Changelog: explain Redis 6 SPOP change. --- 00-RELEASENOTES | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/00-RELEASENOTES b/00-RELEASENOTES index 3572bcc54..9d1670536 100644 --- a/00-RELEASENOTES +++ b/00-RELEASENOTES @@ -114,7 +114,10 @@ Redis 6.0 is mostly a strict superset of 5.0, you should not have any problem upgrading your application from 5.0 to 6.0. However this is a list of small non-backward compatible changes introduced in the 6.0 release: -* Nothing found yet. +* The SPOP command no longer returns null when the set key does not + exist. Now it returns the empty set as it should and as happens when it is + called with a 0 argument. This is technically a fix, however it changes the + old behavior. -------------------------------------------------------------------------------- From e3c1f43952e324ffc3c0d940130b1eda696708e2 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 27 Feb 2020 18:21:12 +0100 Subject: [PATCH 105/225] Show Redis version when not understanding a config directive. This makes simpler to give people help when posting such kind of errors in the mailing list or other help forums, because sometimes the directive looks well spelled, but the version of Redis they are using is not able to support it. --- src/config.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/config.c b/src/config.c index aeb2fea7e..fd04b7c87 100644 --- a/src/config.c +++ b/src/config.c @@ -518,7 +518,8 @@ void loadServerConfigFromString(char *config) { return; loaderr: - fprintf(stderr, "\n*** FATAL CONFIG FILE ERROR ***\n"); + fprintf(stderr, "\n*** FATAL CONFIG FILE ERROR (Redis %s) ***\n", + REDIS_VERSION); fprintf(stderr, "Reading the configuration file, at line %d\n", linenum); fprintf(stderr, ">>> '%s'\n", lines[i]); fprintf(stderr, "%s\n", err); From df152b0ce7260e1edc80bce5c7e0a853cc66cbab Mon Sep 17 00:00:00 2001 From: Guy Benoish Date: Wed, 18 Dec 2019 12:27:03 +0530 Subject: [PATCH 106/225] streamReplyWithRangeFromConsumerPEL: Redundant streamDecodeID --- src/t_stream.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/t_stream.c b/src/t_stream.c index e1efc6bca..557d1d642 100644 --- a/src/t_stream.c +++ b/src/t_stream.c @@ -1068,9 +1068,7 @@ size_t streamReplyWithRangeFromConsumerPEL(client *c, stream *s, streamID *start * by the user by other means. In that case we signal it emitting * the ID but then a NULL entry for the fields. */ addReplyArrayLen(c,2); - streamID id; - streamDecodeID(ri.key,&id); - addReplyStreamID(c,&id); + addReplyStreamID(c,&thisid); addReplyNullArray(c); } else { streamNACK *nack = ri.data; From afe0b16c02b48ff7af2a616fdfdd54b37f64758d Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Sun, 16 Feb 2020 15:43:19 +0200 Subject: [PATCH 107/225] module api docs for aux_save and aux_load --- src/module.c | 6 ++++++ src/rdb.c | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/module.c b/src/module.c index cce3d1c11..17af5336e 100644 --- a/src/module.c +++ b/src/module.c @@ -3529,6 +3529,8 @@ void moduleTypeNameByID(char *name, uint64_t moduleid) { * // Optional fields * .digest = myType_DigestCallBack, * .mem_usage = myType_MemUsageCallBack, + * .aux_load = myType_AuxRDBLoadCallBack, + * .aux_save = myType_AuxRDBSaveCallBack, * } * * * **rdb_load**: A callback function pointer that loads data from RDB files. @@ -3536,6 +3538,10 @@ void moduleTypeNameByID(char *name, uint64_t moduleid) { * * **aof_rewrite**: A callback function pointer that rewrites data as commands. * * **digest**: A callback function pointer that is used for `DEBUG DIGEST`. * * **free**: A callback function pointer that can free a type value. + * * **aux_save**: A callback function pointer that saves out of keyspace data to RDB files. + * 'when' argument is either REDISMODULE_AUX_BEFORE_RDB or REDISMODULE_AUX_AFTER_RDB. + * * **aux_load**: A callback function pointer that loads out of keyspace data from RDB files. + * Similar to aux_save, returns REDISMODULE_OK on success, and ERR otherwise. * * The **digest* and **mem_usage** methods should currently be omitted since * they are not yet implemented inside the Redis modules core. diff --git a/src/rdb.c b/src/rdb.c index 61265433d..cbcea96c6 100644 --- a/src/rdb.c +++ b/src/rdb.c @@ -2195,7 +2195,7 @@ int rdbLoadRio(rio *rdb, int rdbflags, rdbSaveInfo *rsi) { io.ver = 2; /* Call the rdb_load method of the module providing the 10 bit * encoding version in the lower 10 bits of the module ID. */ - if (mt->aux_load(&io,moduleid&1023, when) || io.error) { + if (mt->aux_load(&io,moduleid&1023, when) != REDISMODULE_OK || io.error) { moduleTypeNameByID(name,moduleid); serverLog(LL_WARNING,"The RDB file contains module AUX data for the module type '%s', that the responsible module is not able to load. Check for modules log above for additional clues.", name); exit(1); From 3144a278ddcbf86a73ce018bc510b5bc2222758f Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Thu, 6 Feb 2020 14:53:33 +0200 Subject: [PATCH 108/225] add no_auth to COMMAND INFO --- src/server.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/server.c b/src/server.c index 22c81070c..bb8b3b103 100644 --- a/src/server.c +++ b/src/server.c @@ -3773,6 +3773,7 @@ void addReplyCommand(client *c, struct redisCommand *cmd) { flagcount += addReplyCommandFlag(c,cmd,CMD_SKIP_SLOWLOG, "skip_slowlog"); flagcount += addReplyCommandFlag(c,cmd,CMD_ASKING, "asking"); flagcount += addReplyCommandFlag(c,cmd,CMD_FAST, "fast"); + flagcount += addReplyCommandFlag(c,cmd,CMD_NO_AUTH, "no_auth"); if ((cmd->getkeys_proc && !(cmd->flags & CMD_MODULE)) || cmd->flags & CMD_MODULE_GETKEYS) { From 650484604c3e29c9da42c6b8d8a092f46a85e68e Mon Sep 17 00:00:00 2001 From: Guy Benoish Date: Tue, 4 Feb 2020 19:28:09 +0530 Subject: [PATCH 109/225] Add RM_CreateStringFromDouble --- src/module.c | 12 ++++++++++++ src/redismodule.h | 2 ++ 2 files changed, 14 insertions(+) diff --git a/src/module.c b/src/module.c index 17af5336e..b00118681 100644 --- a/src/module.c +++ b/src/module.c @@ -1042,6 +1042,17 @@ RedisModuleString *RM_CreateStringFromLongLong(RedisModuleCtx *ctx, long long ll return RM_CreateString(ctx,buf,len); } +/* Like RedisModule_CreatString(), but creates a string starting from a double + * integer instead of taking a buffer and its length. + * + * The returned string must be released with RedisModule_FreeString() or by + * enabling automatic memory management. */ +RedisModuleString *RM_CreateStringFromDouble(RedisModuleCtx *ctx, double d) { + char buf[128]; + size_t len = d2string(buf,sizeof(buf),d); + return RM_CreateString(ctx,buf,len); +} + /* Like RedisModule_CreatString(), but creates a string starting from a long * double. * @@ -7690,6 +7701,7 @@ void moduleRegisterCoreAPI(void) { REGISTER_API(CreateStringFromCallReply); REGISTER_API(CreateString); REGISTER_API(CreateStringFromLongLong); + REGISTER_API(CreateStringFromDouble); REGISTER_API(CreateStringFromLongDouble); REGISTER_API(CreateStringFromString); REGISTER_API(CreateStringPrintf); diff --git a/src/redismodule.h b/src/redismodule.h index e74611f13..a43443f13 100644 --- a/src/redismodule.h +++ b/src/redismodule.h @@ -467,6 +467,7 @@ size_t REDISMODULE_API_FUNC(RedisModule_CallReplyLength)(RedisModuleCallReply *r RedisModuleCallReply *REDISMODULE_API_FUNC(RedisModule_CallReplyArrayElement)(RedisModuleCallReply *reply, size_t idx); RedisModuleString *REDISMODULE_API_FUNC(RedisModule_CreateString)(RedisModuleCtx *ctx, const char *ptr, size_t len); RedisModuleString *REDISMODULE_API_FUNC(RedisModule_CreateStringFromLongLong)(RedisModuleCtx *ctx, long long ll); +RedisModuleString *REDISMODULE_API_FUNC(RedisModule_CreateStringFromDouble)(RedisModuleCtx *ctx, double d); RedisModuleString *REDISMODULE_API_FUNC(RedisModule_CreateStringFromLongDouble)(RedisModuleCtx *ctx, long double ld, int humanfriendly); RedisModuleString *REDISMODULE_API_FUNC(RedisModule_CreateStringFromString)(RedisModuleCtx *ctx, const RedisModuleString *str); RedisModuleString *REDISMODULE_API_FUNC(RedisModule_CreateStringPrintf)(RedisModuleCtx *ctx, const char *fmt, ...); @@ -726,6 +727,7 @@ static int RedisModule_Init(RedisModuleCtx *ctx, const char *name, int ver, int REDISMODULE_GET_API(CreateStringFromCallReply); REDISMODULE_GET_API(CreateString); REDISMODULE_GET_API(CreateStringFromLongLong); + REDISMODULE_GET_API(CreateStringFromDouble); REDISMODULE_GET_API(CreateStringFromLongDouble); REDISMODULE_GET_API(CreateStringFromString); REDISMODULE_GET_API(CreateStringPrintf); From fff6b26ae3c6144876c5971ce7b292b7da96365a Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Wed, 5 Feb 2020 18:24:14 +0200 Subject: [PATCH 110/225] RM_Scan disable dict rehashing The callback approach we took is very efficient, the module can do any filtering of keys without building any list and cloning strings, it can also read data from the key's value. but if the user tries to re-open the key, or any other key, this can cause dict re-hashing (dictFind does that), and that's very bad to do from inside dictScan. this commit protects the dict from doing any rehashing during scan, but also warns the user not to attempt any writes or command calls from within the callback, for fear of unexpected side effects and crashes. --- src/dict.c | 7 +++++++ src/module.c | 20 ++++++++++++++------ 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/src/dict.c b/src/dict.c index 106467ef7..93e6c39a7 100644 --- a/src/dict.c +++ b/src/dict.c @@ -871,6 +871,10 @@ unsigned long dictScan(dict *d, if (dictSize(d) == 0) return 0; + /* Having a safe iterator means no rehashing can happen, see _dictRehashStep. + * This is needed in case the scan callback tries to do dictFind or alike. */ + d->iterators++; + if (!dictIsRehashing(d)) { t0 = &(d->ht[0]); m0 = t0->sizemask; @@ -937,6 +941,9 @@ unsigned long dictScan(dict *d, } while (v & (m0 ^ m1)); } + /* undo the ++ at the top */ + d->iterators--; + return v; } diff --git a/src/module.c b/src/module.c index b00118681..b821f0c31 100644 --- a/src/module.c +++ b/src/module.c @@ -6553,9 +6553,13 @@ void RM_ScanCursorDestroy(RedisModuleScanCursor *cursor) { * } * RedisModule_ScanCursorDestroy(c); * - * The function will return 1 if there are more elements to scan and 0 otherwise, - * possibly setting errno if the call failed. - * It is also possible to restart and existing cursor using RM_CursorRestart. */ + * The function will return 1 if there are more elements to scan and 0 otherwise, + * possibly setting errno if the call failed. + * It is also possible to restart and existing cursor using RM_CursorRestart. + * + * NOTE: You must avoid doing any database changes from within the callback, you should avoid any + * RedisModule_OpenKey or RedisModule_Call, if you need to do these, you need to keep the key name + * and do any work you need to do after the call to Scan returns. */ int RM_Scan(RedisModuleCtx *ctx, RedisModuleScanCursor *cursor, RedisModuleScanCB fn, void *privdata) { if (cursor->done) { errno = ENOENT; @@ -6633,9 +6637,13 @@ static void moduleScanKeyCallback(void *privdata, const dictEntry *de) { * RedisModule_CloseKey(key); * RedisModule_ScanCursorDestroy(c); * - * The function will return 1 if there are more elements to scan and 0 otherwise, - * possibly setting errno if the call failed. - * It is also possible to restart and existing cursor using RM_CursorRestart. */ + * The function will return 1 if there are more elements to scan and 0 otherwise, + * possibly setting errno if the call failed. + * It is also possible to restart and existing cursor using RM_CursorRestart. + * + * NOTE: You must avoid doing any database changes from within the callback, you should avoid any + * RedisModule_OpenKey or RedisModule_Call, if you need to do these, you need to keep the field name + * and do any work you need to do after the call to Scan returns. */ int RM_ScanKey(RedisModuleKey *key, RedisModuleScanCursor *cursor, RedisModuleScanKeyCB fn, void *privdata) { if (key == NULL || key->value == NULL) { errno = EINVAL; From c5319612b4879202d9f40b02c895561736f4af92 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 28 Feb 2020 18:06:30 +0100 Subject: [PATCH 111/225] Modules: more details in RM_Scan API top comment. --- src/module.c | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/src/module.c b/src/module.c index b821f0c31..47371fb32 100644 --- a/src/module.c +++ b/src/module.c @@ -6557,9 +6557,21 @@ void RM_ScanCursorDestroy(RedisModuleScanCursor *cursor) { * possibly setting errno if the call failed. * It is also possible to restart and existing cursor using RM_CursorRestart. * - * NOTE: You must avoid doing any database changes from within the callback, you should avoid any - * RedisModule_OpenKey or RedisModule_Call, if you need to do these, you need to keep the key name - * and do any work you need to do after the call to Scan returns. */ + * IMPORTANT: This API is very similar to the Redis SCAN command from the + * point of view of the guarantees it provides. This means that the API + * may report duplicated keys, but guarantees to report at least one time + * every key that was there from the start to the end of the scanning process. + * + * NOTE: If you do database changes within the callback, you should be aware + * that the internal state of the database may change. For instance it is safe + * to delete or modify the current key, but may not be safe to delete any + * other key. + * Moreover playing with the Redis keyspace while iterating may have the + * effect of returning more duplicates. A safe pattern is to store the keys + * names you want to modify elsewhere, and perform the actions on the keys + * later when the iteration is complete. Howerver this can cost a lot of + * memory, so it may make sense to just operate on the current key when + * possible during the iteration, given that this is safe. */ int RM_Scan(RedisModuleCtx *ctx, RedisModuleScanCursor *cursor, RedisModuleScanCB fn, void *privdata) { if (cursor->done) { errno = ENOENT; @@ -6641,9 +6653,13 @@ static void moduleScanKeyCallback(void *privdata, const dictEntry *de) { * possibly setting errno if the call failed. * It is also possible to restart and existing cursor using RM_CursorRestart. * - * NOTE: You must avoid doing any database changes from within the callback, you should avoid any - * RedisModule_OpenKey or RedisModule_Call, if you need to do these, you need to keep the field name - * and do any work you need to do after the call to Scan returns. */ + * NOTE: Certain operations are unsafe while iterating the object. For instance + * while the API guarantees to return at least one time all the elements that + * are present in the data structure consistently from the start to the end + * of the iteration (see HSCAN and similar commands documentation), the more + * you play with the elements, the more duplicates you may get. In general + * deleting the current element of the data structure is safe, while removing + * the key you are iterating is not safe. */ int RM_ScanKey(RedisModuleKey *key, RedisModuleScanCursor *cursor, RedisModuleScanKeyCB fn, void *privdata) { if (key == NULL || key->value == NULL) { errno = EINVAL; From edc0ed141573cfa62fcc890b9f0979de6d7947a5 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 28 Feb 2020 18:09:46 +0100 Subject: [PATCH 112/225] Modules: reformat RM_Scan() top comment a bit. --- src/module.c | 33 +++++++++++++++++++++------------ 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/src/module.c b/src/module.c index 47371fb32..bbd54082c 100644 --- a/src/module.c +++ b/src/module.c @@ -6526,24 +6526,32 @@ void RM_ScanCursorDestroy(RedisModuleScanCursor *cursor) { zfree(cursor); } -/* Scan api that allows a module to scan all the keys and value in the selected db. +/* Scan API that allows a module to scan all the keys and value in + * the selected db. * * Callback for scan implementation. - * void scan_callback(RedisModuleCtx *ctx, RedisModuleString *keyname, RedisModuleKey *key, void *privdata); - * - ctx - the redis module context provided to for the scan. - * - keyname - owned by the caller and need to be retained if used after this function. - * - key - holds info on the key and value, it is provided as best effort, in some cases it might - * be NULL, in which case the user should (can) use RedisModule_OpenKey (and CloseKey too). - * when it is provided, it is owned by the caller and will be free when the callback returns. - * - privdata - the user data provided to RedisModule_Scan. + * void scan_callback(RedisModuleCtx *ctx, RedisModuleString *keyname, + * RedisModuleKey *key, void *privdata); + * ctx - the redis module context provided to for the scan. + * keyname - owned by the caller and need to be retained if used after this + * function. + * + * key - holds info on the key and value, it is provided as best effort, in + * some cases it might be NULL, in which case the user should (can) use + * RedisModule_OpenKey (and CloseKey too). + * when it is provided, it is owned by the caller and will be free when the + * callback returns. + * + * privdata - the user data provided to RedisModule_Scan. * * The way it should be used: * RedisModuleCursor *c = RedisModule_ScanCursorCreate(); * while(RedisModule_Scan(ctx, c, callback, privateData)); * RedisModule_ScanCursorDestroy(c); * - * It is also possible to use this API from another thread while the lock is acquired durring - * the actuall call to RM_Scan: + * It is also possible to use this API from another thread while the lock + * is acquired durring the actuall call to RM_Scan: + * * RedisModuleCursor *c = RedisModule_ScanCursorCreate(); * RedisModule_ThreadSafeContextLock(ctx); * while(RedisModule_Scan(ctx, c, callback, privateData)){ @@ -6553,8 +6561,9 @@ void RM_ScanCursorDestroy(RedisModuleScanCursor *cursor) { * } * RedisModule_ScanCursorDestroy(c); * - * The function will return 1 if there are more elements to scan and 0 otherwise, - * possibly setting errno if the call failed. + * The function will return 1 if there are more elements to scan and + * 0 otherwise, possibly setting errno if the call failed. + * * It is also possible to restart and existing cursor using RM_CursorRestart. * * IMPORTANT: This API is very similar to the Redis SCAN command from the From 10e71b3d01eb2e5d37227ca1f79ee6623fae62f2 Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Wed, 5 Feb 2020 18:06:33 +0200 Subject: [PATCH 113/225] Optimize temporary memory allocations for getKeysFromCommand mechanism now that we may use it more often (ACL), these excessive calls to malloc and free can become an overhead. --- src/db.c | 41 +++++++++++++++++++++++++++++++---------- 1 file changed, 31 insertions(+), 10 deletions(-) diff --git a/src/db.c b/src/db.c index 8a0242d9e..6c5d4b4d4 100644 --- a/src/db.c +++ b/src/db.c @@ -1305,6 +1305,8 @@ int expireIfNeeded(redisDb *db, robj *key) { /* ----------------------------------------------------------------------------- * API to get key arguments from commands * ---------------------------------------------------------------------------*/ +#define MAX_KEYS_BUFFER 65536 +static int getKeysTempBuffer[MAX_KEYS_BUFFER]; /* The base case is to use the keys position as given in the command table * (firstkey, lastkey, step). */ @@ -1319,7 +1321,12 @@ int *getKeysUsingCommandTable(struct redisCommand *cmd,robj **argv, int argc, in last = cmd->lastkey; if (last < 0) last = argc+last; - keys = zmalloc(sizeof(int)*((last - cmd->firstkey)+1)); + + int count = ((last - cmd->firstkey)+1); + keys = getKeysTempBuffer; + if (count > MAX_KEYS_BUFFER) + keys = zmalloc(sizeof(int)*count); + for (j = cmd->firstkey; j <= last; j += cmd->keystep) { if (j >= argc) { /* Modules commands, and standard commands with a not fixed number @@ -1329,7 +1336,7 @@ int *getKeysUsingCommandTable(struct redisCommand *cmd,robj **argv, int argc, in * return no keys and expect the command implementation to report * an arity or syntax error. */ if (cmd->flags & CMD_MODULE || cmd->arity < 0) { - zfree(keys); + getKeysFreeResult(keys); *numkeys = 0; return NULL; } else { @@ -1365,7 +1372,8 @@ int *getKeysFromCommand(struct redisCommand *cmd, robj **argv, int argc, int *nu /* Free the result of getKeysFromCommand. */ void getKeysFreeResult(int *result) { - zfree(result); + if (result != getKeysTempBuffer) + zfree(result); } /* Helper function to extract keys from following commands: @@ -1386,7 +1394,9 @@ int *zunionInterGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *nu /* Keys in z{union,inter}store come from two places: * argv[1] = storage key, * argv[3...n] = keys to intersect */ - keys = zmalloc(sizeof(int)*(num+1)); + keys = getKeysTempBuffer; + if (num+1>MAX_KEYS_BUFFER) + keys = zmalloc(sizeof(int)*(num+1)); /* Add all key positions for argv[3...n] to keys[] */ for (i = 0; i < num; i++) keys[i] = 3+i; @@ -1412,7 +1422,10 @@ int *evalGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numkeys) return NULL; } - keys = zmalloc(sizeof(int)*num); + keys = getKeysTempBuffer; + if (num>MAX_KEYS_BUFFER) + keys = zmalloc(sizeof(int)*num); + *numkeys = num; /* Add all key positions for argv[3...n] to keys[] */ @@ -1433,7 +1446,7 @@ int *sortGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numkeys) UNUSED(cmd); num = 0; - keys = zmalloc(sizeof(int)*2); /* Alloc 2 places for the worst case. */ + keys = getKeysTempBuffer; /* Alloc 2 places for the worst case. */ keys[num++] = 1; /* is always present. */ @@ -1491,7 +1504,10 @@ int *migrateGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numkey } } - keys = zmalloc(sizeof(int)*num); + keys = getKeysTempBuffer; + if (num>MAX_KEYS_BUFFER) + keys = zmalloc(sizeof(int)*num); + for (i = 0; i < num; i++) keys[i] = first+i; *numkeys = num; return keys; @@ -1524,7 +1540,9 @@ int *georadiusGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numk * argv[1] = key, * argv[5...n] = stored key if present */ - keys = zmalloc(sizeof(int) * num); + keys = getKeysTempBuffer; + if (num>MAX_KEYS_BUFFER) + keys = zmalloc(sizeof(int) * num); /* Add all key positions to keys[] */ keys[0] = 1; @@ -1542,7 +1560,7 @@ int *memoryGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numkeys UNUSED(cmd); if (argc >= 3 && !strcasecmp(argv[1]->ptr,"usage")) { - keys = zmalloc(sizeof(int) * 1); + keys = getKeysTempBuffer; keys[0] = 2; *numkeys = 1; return keys; @@ -1589,7 +1607,10 @@ int *xreadGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numkeys) num /= 2; /* We have half the keys as there are arguments because there are also the IDs, one per key. */ - keys = zmalloc(sizeof(int) * num); + keys = getKeysTempBuffer; + if (num>MAX_KEYS_BUFFER) + keys = zmalloc(sizeof(int) * num); + for (i = streams_pos+1; i < argc-num; i++) keys[i-streams_pos-1] = i; *numkeys = num; return keys; From 07dc1b42fb9910bbc9c2c7e22f2feff9183bd0f7 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 2 Mar 2020 16:49:11 +0100 Subject: [PATCH 114/225] Use a smaller getkeys global buffer. The idea is that very few commands have a lot of keys, and when this happens the allocation time becomes neglegible. --- src/db.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/db.c b/src/db.c index 6c5d4b4d4..04e26c33b 100644 --- a/src/db.c +++ b/src/db.c @@ -1305,7 +1305,7 @@ int expireIfNeeded(redisDb *db, robj *key) { /* ----------------------------------------------------------------------------- * API to get key arguments from commands * ---------------------------------------------------------------------------*/ -#define MAX_KEYS_BUFFER 65536 +#define MAX_KEYS_BUFFER 256 static int getKeysTempBuffer[MAX_KEYS_BUFFER]; /* The base case is to use the keys position as given in the command table From 7a23b94559cca33c219dbdc717005ef14a55a6cf Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 4 Mar 2020 11:19:55 +0100 Subject: [PATCH 115/225] Log RDB deletion in persistence-less instances. --- src/replication.c | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/src/replication.c b/src/replication.c index 8a05b9696..acecdd098 100644 --- a/src/replication.c +++ b/src/replication.c @@ -944,6 +944,8 @@ void removeRDBUsedToSyncReplicas(void) { } } if (delrdb) { + serverLog(LL_NOTICE,"Removing the RDB file used to feed replicas " + "in a persistence-less instance"); RDBGeneratedByReplication = 0; bg_unlink(server.rdb_filename); } @@ -1707,14 +1709,25 @@ void readSyncBulkPayload(connection *conn) { "Failed trying to load the MASTER synchronization " "DB from disk"); cancelReplicationHandshake(); - if (allPersistenceDisabled()) bg_unlink(server.rdb_filename); + if (allPersistenceDisabled()) { + serverLog(LL_NOTICE,"Removing the RDB file obtained from " + "the master. This replica has persistence " + "disabled"); + bg_unlink(server.rdb_filename); + } /* Note that there's no point in restarting the AOF on sync failure, it'll be restarted when sync succeeds or replica promoted. */ return; } /* Cleanup. */ - if (allPersistenceDisabled()) bg_unlink(server.rdb_filename); + if (allPersistenceDisabled()) { + serverLog(LL_NOTICE,"Removing the RDB file obtained from " + "the master. This replica has persistence " + "disabled"); + bg_unlink(server.rdb_filename); + } + zfree(server.repl_transfer_tmpfile); close(server.repl_transfer_fd); server.repl_transfer_fd = -1; From be4bc1a5be26a7fde2fd05acd8187f5f0ed59f25 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 3 Mar 2020 14:58:11 +0100 Subject: [PATCH 116/225] Remove RDB files used for replication in persistence-less instances. --- src/replication.c | 48 ++++++++++++++++++++++++++++++++++++++++++++++- src/server.c | 8 ++++++++ src/server.h | 1 + 3 files changed, 56 insertions(+), 1 deletion(-) diff --git a/src/replication.c b/src/replication.c index c497051c8..e71b269d8 100644 --- a/src/replication.c +++ b/src/replication.c @@ -45,6 +45,11 @@ void replicationSendAck(void); void putSlaveOnline(client *slave); int cancelReplicationHandshake(void); +/* We take a global flag to remember if this instance generated an RDB + * because of replication, so that we can remove the RDB file in case + * the instance is configured to have no persistence. */ +int RDBGeneratedByReplication = 0; + /* --------------------------- Utility functions ---------------------------- */ /* Return the pointer to a string representing the slave ip:listening_port @@ -591,6 +596,10 @@ int startBgsaveForReplication(int mincapa) { retval = C_ERR; } + /* If we succeeded to start a BGSAVE with disk target, let's remember + * this fact, so that we can later delete the file if needed. */ + if (retval == C_OK && !socket_target) RDBGeneratedByReplication = 1; + /* If we failed to BGSAVE, remove the slaves waiting for a full * resynchronization from the list of slaves, inform them with * an error about what happened, close the connection ASAP. */ @@ -883,6 +892,36 @@ void putSlaveOnline(client *slave) { replicationGetSlaveName(slave)); } +/* We call this function periodically to remove an RDB file that was + * generated because of replication, in an instance that is otherwise + * without any persistence. We don't want instances without persistence + * to take RDB files around, this violates certain policies in certain + * environments. */ +void removeRDBUsedToSyncReplicas(void) { + if (allPersistenceDisabled() && RDBGeneratedByReplication) { + client *slave; + listNode *ln; + listIter li; + + int delrdb = 1; + listRewind(server.slaves,&li); + while((ln = listNext(&li))) { + slave = ln->value; + if (slave->replstate == SLAVE_STATE_WAIT_BGSAVE_START || + slave->replstate == SLAVE_STATE_WAIT_BGSAVE_END || + slave->replstate == SLAVE_STATE_SEND_BULK) + { + delrdb = 0; + break; /* No need to check the other replicas. */ + } + } + if (delrdb) { + RDBGeneratedByReplication = 0; + unlink(server.rdb_filename); + } + } +} + void sendBulkToSlave(connection *conn) { client *slave = connGetPrivateData(conn); char buf[PROTO_IOBUF_LEN]; @@ -894,7 +933,8 @@ void sendBulkToSlave(connection *conn) { if (slave->replpreamble) { nwritten = connWrite(conn,slave->replpreamble,sdslen(slave->replpreamble)); if (nwritten == -1) { - serverLog(LL_VERBOSE,"Write error sending RDB preamble to replica: %s", + serverLog(LL_VERBOSE, + "Write error sending RDB preamble to replica: %s", connGetLastError(conn)); freeClient(slave); return; @@ -1639,12 +1679,14 @@ void readSyncBulkPayload(connection *conn) { "Failed trying to load the MASTER synchronization " "DB from disk"); cancelReplicationHandshake(); + if (allPersistenceDisabled()) unlink(server.rdb_filename); /* Note that there's no point in restarting the AOF on sync failure, it'll be restarted when sync succeeds or replica promoted. */ return; } /* Cleanup. */ + if (allPersistenceDisabled()) unlink(server.rdb_filename); zfree(server.repl_transfer_tmpfile); close(server.repl_transfer_fd); server.repl_transfer_fd = -1; @@ -3149,6 +3191,10 @@ void replicationCron(void) { } } + /* Remove the RDB file used for replication if Redis is not running + * with any persistence. */ + removeRDBUsedToSyncReplicas(); + /* Refresh the number of slaves with lag <= min-slaves-max-lag. */ refreshGoodSlavesCount(); replication_cron_loops++; /* Incremented with frequency 1 HZ. */ diff --git a/src/server.c b/src/server.c index bb8b3b103..a6d4b357e 100644 --- a/src/server.c +++ b/src/server.c @@ -1455,12 +1455,20 @@ void updateDictResizePolicy(void) { dictDisableResize(); } +/* Return true if there are no active children processes doing RDB saving, + * AOF rewriting, or some side process spawned by a loaded module. */ int hasActiveChildProcess() { return server.rdb_child_pid != -1 || server.aof_child_pid != -1 || server.module_child_pid != -1; } +/* Return true if this instance has persistence completely turned off: + * both RDB and AOF are disabled. */ +int allPersistenceDisabled(void) { + return server.saveparamslen == 0 && server.aof_state == AOF_OFF; +} + /* ======================= Cron: called every 100 ms ======================== */ /* Add a sample to the operations per second array of samples. */ diff --git a/src/server.h b/src/server.h index 87c293c26..5763671e4 100644 --- a/src/server.h +++ b/src/server.h @@ -1786,6 +1786,7 @@ void loadingProgress(off_t pos); void stopLoading(int success); void startSaving(int rdbflags); void stopSaving(int success); +int allPersistenceDisabled(void); #define DISK_ERROR_TYPE_AOF 1 /* Don't accept writes: AOF errors. */ #define DISK_ERROR_TYPE_RDB 2 /* Don't accept writes: RDB errors. */ From baaf869fc3e138cfcb4cfda09f09fd3c87c8f924 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 4 Mar 2020 11:10:54 +0100 Subject: [PATCH 117/225] Introduce bg_unlink(). --- src/replication.c | 34 +++++++++++++++++++++++++++++++--- 1 file changed, 31 insertions(+), 3 deletions(-) diff --git a/src/replication.c b/src/replication.c index e71b269d8..8a05b9696 100644 --- a/src/replication.c +++ b/src/replication.c @@ -79,6 +79,34 @@ char *replicationGetSlaveName(client *c) { return buf; } +/* Plain unlink() can block for quite some time in order to actually apply + * the file deletion to the filesystem. This call removes the file in a + * background thread instead. We actually just do close() in the thread, + * by using the fact that if there is another instance of the same file open, + * the foreground unlink() will not really do anything, and deleting the + * file will only happen once the last reference is lost. */ +int bg_unlink(const char *filename) { + int fd = open(filename,O_RDONLY|O_NONBLOCK); + if (fd == -1) { + /* Can't open the file? Fall back to unlinking in the main thread. */ + return unlink(filename); + } else { + /* The following unlink() will not do anything since file + * is still open. */ + int retval = unlink(filename); + if (retval == -1) { + /* If we got an unlink error, we just return it, closing the + * new reference we have to the file. */ + int old_errno = errno; + close(fd); /* This would overwrite our errno. So we saved it. */ + errno = old_errno; + return -1; + } + bioCreateBackgroundJob(BIO_CLOSE_FILE,(void*)(long)fd,NULL,NULL); + return 0; /* Success. */ + } +} + /* ---------------------------------- MASTER -------------------------------- */ void createReplicationBacklog(void) { @@ -917,7 +945,7 @@ void removeRDBUsedToSyncReplicas(void) { } if (delrdb) { RDBGeneratedByReplication = 0; - unlink(server.rdb_filename); + bg_unlink(server.rdb_filename); } } } @@ -1679,14 +1707,14 @@ void readSyncBulkPayload(connection *conn) { "Failed trying to load the MASTER synchronization " "DB from disk"); cancelReplicationHandshake(); - if (allPersistenceDisabled()) unlink(server.rdb_filename); + if (allPersistenceDisabled()) bg_unlink(server.rdb_filename); /* Note that there's no point in restarting the AOF on sync failure, it'll be restarted when sync succeeds or replica promoted. */ return; } /* Cleanup. */ - if (allPersistenceDisabled()) unlink(server.rdb_filename); + if (allPersistenceDisabled()) bg_unlink(server.rdb_filename); zfree(server.repl_transfer_tmpfile); close(server.repl_transfer_fd); server.repl_transfer_fd = -1; From a20303c62339fcaa07bd7872df7fe05d9fe54b1f Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 4 Mar 2020 12:55:49 +0100 Subject: [PATCH 118/225] Check that the file exists in removeRDBUsedToSyncReplicas(). --- src/replication.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/replication.c b/src/replication.c index acecdd098..20666bd20 100644 --- a/src/replication.c +++ b/src/replication.c @@ -944,10 +944,14 @@ void removeRDBUsedToSyncReplicas(void) { } } if (delrdb) { - serverLog(LL_NOTICE,"Removing the RDB file used to feed replicas " - "in a persistence-less instance"); - RDBGeneratedByReplication = 0; - bg_unlink(server.rdb_filename); + struct stat sb; + if (lstat(server.rdb_filename,&sb) != -1) { + RDBGeneratedByReplication = 0; + serverLog(LL_NOTICE, + "Removing the RDB file used to feed replicas " + "in a persistence-less instance"); + bg_unlink(server.rdb_filename); + } } } } From 127e09bca16a23e987820dc45f5cdb8a6fa6a3fa Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 4 Mar 2020 17:44:21 +0100 Subject: [PATCH 119/225] Make sync RDB deletion configurable. Default to no. --- src/config.c | 1 + src/replication.c | 23 +++++++++++++++++++---- src/server.h | 2 ++ 3 files changed, 22 insertions(+), 4 deletions(-) diff --git a/src/config.c b/src/config.c index fd04b7c87..211b6d003 100644 --- a/src/config.c +++ b/src/config.c @@ -2084,6 +2084,7 @@ standardConfig configs[] = { createBoolConfig("always-show-logo", NULL, IMMUTABLE_CONFIG, server.always_show_logo, 0, NULL, NULL), createBoolConfig("protected-mode", NULL, MODIFIABLE_CONFIG, server.protected_mode, 1, NULL, NULL), createBoolConfig("rdbcompression", NULL, MODIFIABLE_CONFIG, server.rdb_compression, 1, NULL, NULL), + createBoolConfig("rdb-del-sync-files", NULL, MODIFIABLE_CONFIG, server.rdb_del_sync_files, 0, NULL, NULL), createBoolConfig("activerehashing", NULL, MODIFIABLE_CONFIG, server.activerehashing, 1, NULL, NULL), createBoolConfig("stop-writes-on-bgsave-error", NULL, MODIFIABLE_CONFIG, server.stop_writes_on_bgsave_err, 1, NULL, NULL), createBoolConfig("dynamic-hz", NULL, MODIFIABLE_CONFIG, server.dynamic_hz, 1, NULL, NULL), /* Adapt hz to # of clients.*/ diff --git a/src/replication.c b/src/replication.c index 20666bd20..429e1d8a8 100644 --- a/src/replication.c +++ b/src/replication.c @@ -625,8 +625,12 @@ int startBgsaveForReplication(int mincapa) { } /* If we succeeded to start a BGSAVE with disk target, let's remember - * this fact, so that we can later delete the file if needed. */ - if (retval == C_OK && !socket_target) RDBGeneratedByReplication = 1; + * this fact, so that we can later delete the file if needed. Note + * that we don't set the flag to 1 if the feature is disabled, otherwise + * it would never be cleared: the file is not deleted. This way if + * the user enables it later with CONFIG SET, we are fine. */ + if (retval == C_OK && !socket_target && server.rdb_del_sync_files) + RDBGeneratedByReplication = 1; /* If we failed to BGSAVE, remove the slaves waiting for a full * resynchronization from the list of slaves, inform them with @@ -926,6 +930,17 @@ void putSlaveOnline(client *slave) { * to take RDB files around, this violates certain policies in certain * environments. */ void removeRDBUsedToSyncReplicas(void) { + /* If the feature is disabled, return ASAP but also clear the + * RDBGeneratedByReplication flag in case it was set. Otherwise if the + * feature was enabled, but gets disabled later with CONFIG SET, the + * flag may remain set to one: then next time the feature is re-enabled + * via CONFIG SET we have have it set even if no RDB was generated + * because of replication recently. */ + if (!server.rdb_del_sync_files) { + RDBGeneratedByReplication = 0; + return; + } + if (allPersistenceDisabled() && RDBGeneratedByReplication) { client *slave; listNode *ln; @@ -1713,7 +1728,7 @@ void readSyncBulkPayload(connection *conn) { "Failed trying to load the MASTER synchronization " "DB from disk"); cancelReplicationHandshake(); - if (allPersistenceDisabled()) { + if (server.rdb_del_sync_files && allPersistenceDisabled()) { serverLog(LL_NOTICE,"Removing the RDB file obtained from " "the master. This replica has persistence " "disabled"); @@ -1725,7 +1740,7 @@ void readSyncBulkPayload(connection *conn) { } /* Cleanup. */ - if (allPersistenceDisabled()) { + if (server.rdb_del_sync_files && allPersistenceDisabled()) { serverLog(LL_NOTICE,"Removing the RDB file obtained from " "the master. This replica has persistence " "disabled"); diff --git a/src/server.h b/src/server.h index 5763671e4..3c19a17ea 100644 --- a/src/server.h +++ b/src/server.h @@ -1202,6 +1202,8 @@ struct redisServer { char *rdb_filename; /* Name of RDB file */ int rdb_compression; /* Use compression in RDB? */ int rdb_checksum; /* Use RDB checksum? */ + int rdb_del_sync_files; /* Remove RDB files used only for SYNC if + the instance does not use persistence. */ time_t lastsave; /* Unix time of last successful save */ time_t lastbgsave_try; /* Unix time of last attempted bgsave */ time_t rdb_save_time_last; /* Time used by last RDB save run. */ From c2f01d7f9ee70570e0b404f0ef1eb26e2e81fb6e Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 4 Mar 2020 17:58:05 +0100 Subject: [PATCH 120/225] RDB deletion: document it in example redis.conf. --- redis.conf | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/redis.conf b/redis.conf index c04880f32..8609a9f57 100644 --- a/redis.conf +++ b/redis.conf @@ -321,6 +321,19 @@ rdbchecksum yes # The filename where to dump the DB dbfilename dump.rdb +# Remove RDB files used by replication in instances without persistence +# enabled. By default this option is disabled, however there are environments +# where for regulations or other security concerns, RDB files persisted on +# disk by masters in order to feed replicas, or stored on disk by replicas +# in order to load them for the initial synchronization, should be deleted +# ASAP. Note that this option ONLY WORKS in instances that have both AOF +# and RDB persistence disabled, otherwise is completely ignored. +# +# An alternative (and sometimes better) way to obtain the same effect is +# to use diskless replication on both master and replicas instances. However +# in the case of replicas, diskless is not always an option. +rdb-del-sync-files no + # The working directory. # # The DB will be written inside this directory, with the filename specified From fe81d5c8a9aed2ed2a53259fc1754d1d566f49af Mon Sep 17 00:00:00 2001 From: ShooterIT Date: Sat, 29 Feb 2020 18:28:41 +0800 Subject: [PATCH 121/225] Avoid compiler warnings --- src/acl.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/acl.c b/src/acl.c index b046785ff..efe6b96ad 100644 --- a/src/acl.c +++ b/src/acl.c @@ -1830,6 +1830,7 @@ void aclCommand(client *c) { case ACL_DENIED_CMD: reasonstr="command"; break; case ACL_DENIED_KEY: reasonstr="key"; break; case ACL_DENIED_AUTH: reasonstr="auth"; break; + default: reasonstr="unknown"; } addReplyBulkCString(c,reasonstr); From 6ef018785be9fab1d5d6d10d844da7c35c3375e3 Mon Sep 17 00:00:00 2001 From: hwware Date: Sat, 22 Feb 2020 11:38:51 -0500 Subject: [PATCH 122/225] add missing file marco --- src/sdsalloc.h | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/sdsalloc.h b/src/sdsalloc.h index 531d41929..c04ff2a0a 100644 --- a/src/sdsalloc.h +++ b/src/sdsalloc.h @@ -36,7 +36,12 @@ * the include of your alternate allocator if needed (not needed in order * to use the default libc allocator). */ +#ifndef __SDS_ALLOC_H__ +#define __SDS_ALLOC_H__ + #include "zmalloc.h" #define s_malloc zmalloc #define s_realloc zrealloc #define s_free zfree + +#endif From 4af0d7fd94bb18ff793561297717d0b8c4696b7f Mon Sep 17 00:00:00 2001 From: qetu3790 Date: Thu, 23 Jan 2020 17:18:07 +0800 Subject: [PATCH 123/225] Fix not used constant in lru_test_mode. LRU_CYCLE_PERIOD is defined,but not used. --- src/redis-cli.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/redis-cli.c b/src/redis-cli.c index 1d79e0db0..54898f42e 100644 --- a/src/redis-cli.c +++ b/src/redis-cli.c @@ -7737,7 +7737,7 @@ static void LRUTestMode(void) { * to fill the target instance easily. */ start_cycle = mstime(); long long hits = 0, misses = 0; - while(mstime() - start_cycle < 1000) { + while(mstime() - start_cycle < LRU_CYCLE_PERIOD) { /* Write cycle. */ for (j = 0; j < LRU_CYCLE_PIPELINE_SIZE; j++) { char val[6]; From e74e68c84f5eba8013769087c9a46cab811b8417 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 5 Mar 2020 16:00:17 +0100 Subject: [PATCH 124/225] Redis 6 RC2. --- 00-RELEASENOTES | 603 ++++++++++++++++++++++++++++++++++++++++++++++++ src/version.h | 2 +- 2 files changed, 604 insertions(+), 1 deletion(-) diff --git a/00-RELEASENOTES b/00-RELEASENOTES index 9d1670536..ad2648bcb 100644 --- a/00-RELEASENOTES +++ b/00-RELEASENOTES @@ -11,6 +11,609 @@ CRITICAL: There is a critical bug affecting MOST USERS. Upgrade ASAP. SECURITY: There are security fixes in the release. -------------------------------------------------------------------------------- +================================================================================ +Redis 6.0 RC2 Released Thu Mar 05 15:40:53 CET 2020 +================================================================================ + +Upgrade urgency MODERATE: Normal bugfixing release of a non-GA branch. + +Hi Redis users, Redis 6 is approaching and will be released 30th of April. +New release candidates will be released at the end of March, then another +one mid April, to finally reach the GA at the end of April. + +Redis 6 RC2 brings many fixes and new things, especially in the area of +client side caching. This is the list of big changes in this release. As +usually you can find the full list of commits at the end: + +New features and improvements: + +* ACL LOG: log denied commands, keys accesses and authentications. +* Client side caching redesigned. Now we use keys not caching slots. +* Client side caching: Broadcasting mode implemented. +* Client side caching: OPTIN/OUTPUT modes implemented. +* Remove RDB files used for replication in persistence-less instances (option). + +Fixes (only selected ones, see commits for all the fixes): + +* Different fixes to streams in edge cases. +* Fix duplicated CLIENT SETNAME reply because of RESP3 changes. +* Fix crash due to new active expire division by zero. +* Avoid sentinel changes promoted_slave to be its own replica. +* Fix bug on KEYS command where pattern starts with * followed by \x00. +* Threaded I/O: now the main thread is used as well to do I/O. +* Many fixes to modules APIs, and more to come in the next RCs. +* ld2string should fail if string contains \0 in the middle. +* Make the Redis test more reliable. +* Fix SPOP returning nil (see #4709). WARNING: API change. + +qetu3790 in commit 4af0d7fd: + Fix not used constant in lru_test_mode. + 1 file changed, 1 insertion(+), 1 deletion(-) + +hwware in commit 6ef01878: + add missing file marco + 1 file changed, 5 insertions(+) + +ShooterIT in commit fe81d5c8: + Avoid compiler warnings + 1 file changed, 1 insertion(+) + +antirez in commit c2f01d7f: + RDB deletion: document it in example redis.conf. + 1 file changed, 13 insertions(+) + +antirez in commit 127e09bc: + Make sync RDB deletion configurable. Default to no. + 3 files changed, 22 insertions(+), 4 deletions(-) + +antirez in commit a20303c6: + Check that the file exists in removeRDBUsedToSyncReplicas(). + 1 file changed, 8 insertions(+), 4 deletions(-) + +antirez in commit 7a23b945: + Log RDB deletion in persistence-less instances. + 1 file changed, 15 insertions(+), 2 deletions(-) + +antirez in commit baaf869f: + Introduce bg_unlink(). + 1 file changed, 31 insertions(+), 3 deletions(-) + +antirez in commit be4bc1a5: + Remove RDB files used for replication in persistence-less instances. + 3 files changed, 56 insertions(+), 1 deletion(-) + +antirez in commit 07dc1b42: + Use a smaller getkeys global buffer. + 1 file changed, 1 insertion(+), 1 deletion(-) + +Oran Agra in commit 10e71b3d: + Optimize temporary memory allocations for getKeysFromCommand mechanism + 1 file changed, 31 insertions(+), 10 deletions(-) + +antirez in commit edc0ed14: + Modules: reformat RM_Scan() top comment a bit. + 1 file changed, 21 insertions(+), 12 deletions(-) + +antirez in commit c5319612: + Modules: more details in RM_Scan API top comment. + 1 file changed, 22 insertions(+), 6 deletions(-) + +Oran Agra in commit fff6b26a: + RM_Scan disable dict rehashing + 2 files changed, 21 insertions(+), 6 deletions(-) + +Guy Benoish in commit 65048460: + Add RM_CreateStringFromDouble + 2 files changed, 14 insertions(+) + +Oran Agra in commit 3144a278: + add no_auth to COMMAND INFO + 1 file changed, 1 insertion(+) + +Oran Agra in commit afe0b16c: + module api docs for aux_save and aux_load + 2 files changed, 7 insertions(+), 1 deletion(-) + +Guy Benoish in commit df152b0c: + streamReplyWithRangeFromConsumerPEL: Redundant streamDecodeID + 1 file changed, 1 insertion(+), 3 deletions(-) + +antirez in commit e3c1f439: + Show Redis version when not understanding a config directive. + 1 file changed, 2 insertions(+), 1 deletion(-) + +antirez in commit 141c0679: + Changelog: explain Redis 6 SPOP change. + 1 file changed, 4 insertions(+), 1 deletion(-) + +bodong.ybd in commit fe902461: + Fix spop return nil #4709 + 1 file changed, 1 insertion(+), 1 deletion(-) + +antirez in commit 9d4219eb: + Fix SDS misuse in enumConfigSet(). Related to #6778. + 1 file changed, 3 insertions(+), 3 deletions(-) + +antirez in commit 84243064: + Remove useless comment from enumConfigSet(). + 1 file changed, 1 deletion(-) + +Ponnuvel Palaniyappan in commit dafb94db: + Fix a potential overflow with strncpy + 1 file changed, 5 insertions(+), 5 deletions(-) + +antirez in commit ea697b63: + Improve aeDeleteEventLoop() top comment grammar. + 1 file changed, 2 insertions(+), 1 deletion(-) + +wangyuan21 in commit dd479880: + free time event when delete eventloop + 1 file changed, 7 insertions(+) + +srzhao in commit ecf3b2ef: + fix impl of aof-child whitelist SIGUSR1 feature. + 1 file changed, 5 insertions(+), 4 deletions(-) + +meir@redislabs.com in commit 2966132c: + Changed log level for module fork api from 'notice' to 'verbos'. + 1 file changed, 2 insertions(+), 2 deletions(-) + +hwware in commit 7277e5d8: + format fix + 1 file changed, 1 insertion(+), 1 deletion(-) + +hwware in commit 1bb5ee9c: + fix potentical memory leaks + 1 file changed, 4 insertions(+), 1 deletion(-) + +Hengjian Tang in commit 97329733: + modify the read buf size according to the write buf size PROTO_IOBUF_LEN defined before + 1 file changed, 1 insertion(+), 1 deletion(-) + +Ariel in commit 15ea1324: + fix ThreadSafeContext lock/unlock function names + 1 file changed, 2 insertions(+), 2 deletions(-) + +Guy Benoish in commit 4d12c37c: + XREADGROUP should propagate XCALIM/SETID in MULTI/EXEC + 1 file changed, 2 insertions(+), 2 deletions(-) + +Oran Agra in commit 12626ce9: + fix race in module api test for fork + 2 files changed, 2 insertions(+), 3 deletions(-) + +Guy Benoish in commit 2ecab0b6: + Modules: Do not auto-unblock clients if not blocked on keys + 1 file changed, 22 insertions(+), 7 deletions(-) + +Oran Agra in commit 635321d4: + fix github actions failing latency test for active defrag - part 2 + 2 files changed, 5 insertions(+), 4 deletions(-) + +Oran Agra in commit 0b988fa9: + fix github actions failing latency test for active defrag + 2 files changed, 14 insertions(+), 13 deletions(-) + +Oran Agra in commit 60096bc1: + Fix latency sensitivity of new defrag test + 1 file changed, 32 insertions(+), 8 deletions(-) + +antirez in commit b4395426: + Tracking: optin/out implemented. + 3 files changed, 82 insertions(+), 16 deletions(-) + +antirez in commit ef3551d1: + Test engine: experimental change to avoid busy port problems. + 1 file changed, 84 insertions(+), 49 deletions(-) + +antirez in commit 72c05351: + Test engine: detect timeout when checking for Redis startup. + 1 file changed, 11 insertions(+), 1 deletion(-) + +antirez in commit 294c9af4: + Test engine: better tracking of what workers are doing. + 2 files changed, 12 insertions(+), 4 deletions(-) + +hwware in commit ba027079: + add missing subcommand description for debug oom + 1 file changed, 1 insertion(+) + +Guy Benoish in commit 5d0890c0: + Fix memory leak in test_ld_conv + 1 file changed, 4 insertions(+) + +Madelyn Olson in commit d1f22eac: + Give an error message if you specify redirect twice + 1 file changed, 7 insertions(+) + +Madelyn Olson in commit 762fbcb6: + Minor CSC fixes and fixed documentation + 2 files changed, 16 insertions(+), 17 deletions(-) + +Oran Agra in commit 349aa245: + Defrag big lists in portions to avoid latency and freeze + 4 files changed, 350 insertions(+), 34 deletions(-) + +Guy Benoish in commit b4ddc7b7: + XGROUP DESTROY should unblock XREADGROUP with -NOGROUP + 2 files changed, 11 insertions(+) + +hayashier in commit 73806f74: + fix typo from fss to rss + 1 file changed, 2 insertions(+), 2 deletions(-) + +antirez in commit b6129f86: + Test is more complex now, increase default timeout. + 1 file changed, 1 insertion(+), 1 deletion(-) + +antirez in commit f15fb727: + Tracking: fix max-keys configuration directive. + 2 files changed, 2 insertions(+), 2 deletions(-) + +Itamar Haber in commit e374573f: + Fixes segfault on calling trackingGetTotalKeys + 1 file changed, 1 insertion(+) + +antirez in commit 73d47d57: + Signal key as modified when expired on-access. + 1 file changed, 4 insertions(+), 2 deletions(-) + +antirez in commit b7cb28d5: + Tracking: first set of tests for the feature. + 1 file changed, 66 insertions(+) + +antirez in commit 1db72571: + Tracking: fix operators precedence error in bcast check. + 1 file changed, 1 insertion(+), 1 deletion(-) + +antirez in commit fe96e29d: + Tracking: fix behavior when switchinig from normal to BCAST. + 1 file changed, 11 insertions(+), 1 deletion(-) + +antirez in commit f21be1ec: + Tracking: fix sending messages bug + tracking off bug. + 2 files changed, 28 insertions(+), 20 deletions(-) + +antirez in commit 6fb1aa23: + Tracking: BCAST: basic feature now works. + 3 files changed, 55 insertions(+), 40 deletions(-) + +antirez in commit d4fe79a1: + Tracking: BCAST: broadcasting of keys in prefixes implemented. + 2 files changed, 94 insertions(+), 9 deletions(-) + +antirez in commit abb81c63: + Tracking: BCAST: registration in the prefix table. + 3 files changed, 67 insertions(+), 20 deletions(-) + +antirez in commit 77da9608: + Tracking: BCAST: parsing of the options + skeleton. + 4 files changed, 73 insertions(+), 19 deletions(-) + +antirez in commit 3e8c69a9: + Tracking: always reply with an array of keys. + 2 files changed, 10 insertions(+), 3 deletions(-) + +antirez in commit a788c373: + Tracking: minor change of names and new INFO field. + 4 files changed, 11 insertions(+), 4 deletions(-) + +antirez in commit df838927: + Rax.c: populate data field after random walk. + 1 file changed, 1 insertion(+) + +antirez in commit 0517da36: + Tracking: rename INFO field with total items. + 1 file changed, 1 insertion(+), 1 deletion(-) + +antirez in commit 3c16d6b3: + Tracking: first conversion from hashing to key names. + 3 files changed, 84 insertions(+), 114 deletions(-) + +Oran Agra in commit 3b4f1477: + add no-slowlog option to RM_CreateCommand + 1 file changed, 3 insertions(+) + +Khem Raj in commit 5e762d84: + Mark extern definition of SDS_NOINIT in sds.h + 1 file changed, 1 insertion(+), 1 deletion(-) + +lifubang in commit 54f5499a: + correct help info for --user and --pass + 1 file changed, 2 insertions(+), 2 deletions(-) + +Seunghoon Woo in commit 0c952b13: + [FIX] revisit CVE-2015-8080 vulnerability + 1 file changed, 6 insertions(+), 4 deletions(-) + +Guy Benoish in commit dd34f703: + Diskless-load emptyDb-related fixes + 3 files changed, 44 insertions(+), 28 deletions(-) + +lifubang in commit 5e042dbc: + fix ssl flag check for redis-cli + 1 file changed, 10 insertions(+), 9 deletions(-) + +Guy Benoish in commit dcbe8bfa: + Exclude "keymiss" notification from NOTIFY_ALL + 5 files changed, 12 insertions(+), 7 deletions(-) + +Oran Agra in commit 36caf2e4: + update RM_SignalModifiedKey doc comment + 1 file changed, 2 insertions(+), 1 deletion(-) + +Oran Agra in commit 3067352a: + Add handling of short read of module id in rdb + 1 file changed, 4 insertions(+), 1 deletion(-) + +Yossi Gottlieb in commit 9baaf858: + TLS: Update documentation. + 2 files changed, 32 insertions(+), 31 deletions(-) + +Oran Agra in commit 4440133e: + A few non-data commands that should be allowed while loading or stale + 1 file changed, 8 insertions(+), 8 deletions(-) + +Oran Agra in commit c9577941: + Memory leak when bind config is provided twice + 1 file changed, 4 insertions(+) + +Oran Agra in commit 1333a46b: + fix maxmemory config warning + 1 file changed, 3 insertions(+), 2 deletions(-) + +Oran Agra in commit 8e7282eb: + Fix client flags to be int64 in module.c + 1 file changed, 3 insertions(+), 3 deletions(-) + +Oran Agra in commit a678390e: + moduleRDBLoadError, add key name, and use panic rather than exit + 1 file changed, 5 insertions(+), 4 deletions(-) + +Oran Agra in commit 919fbf42: + reduce repeated calls to use_diskless_load + 1 file changed, 3 insertions(+), 4 deletions(-) + +Oran Agra in commit 22e45d46: + freeClientAsync don't lock mutex if there's just one thread + 1 file changed, 6 insertions(+), 1 deletion(-) + +Oran Agra in commit ba289244: + move restartAOFAfterSYNC from replicaofCommand to replicationUnsetMaster + 1 file changed, 4 insertions(+), 3 deletions(-) + +Oran Agra in commit f42ce57d: + stopAppendOnly resets aof_rewrite_scheduled + 1 file changed, 1 insertion(+) + +Oran Agra in commit df096bc9: + add SAVE subcommand to ACL HELP and top comment + 1 file changed, 2 insertions(+) + +Oran Agra in commit a55e5847: + DEBUG HELP - add PROTOCOL + 1 file changed, 3 insertions(+), 2 deletions(-) + +Guy Benoish in commit 5a6cfbf4: + Some refactroing using getClientType instead of CLIENT_SLAVE + 2 files changed, 18 insertions(+), 26 deletions(-) + +Guy Benoish in commit fae306b3: + Fix small bugs related to replica and monitor ambiguity + 2 files changed, 8 insertions(+), 6 deletions(-) + +Yossi Gottlieb in commit 73630966: + TLS: Some redis.conf clarifications. + 1 file changed, 10 insertions(+), 11 deletions(-) + +Oran Agra in commit 488e1947: + config.c verbose error replies for CONFIG SET, like config file parsing + 1 file changed, 31 insertions(+), 97 deletions(-) + +Oran Agra in commit c82ccf06: + memoryGetKeys helper function so that ACL can limit access to keys for MEMORY command + 3 files changed, 18 insertions(+), 1 deletion(-) + +antirez in commit 51c1a9f8: + ACL LOG: make max log entries configurable. + 4 files changed, 19 insertions(+) + +antirez in commit ea1e1b12: + ACL LOG: test for AUTH reason. + 1 file changed, 9 insertions(+) + +antirez in commit 7379c78a: + ACL LOG: log failed auth attempts. + 5 files changed, 34 insertions(+), 12 deletions(-) + +antirez in commit 9f6e84f6: + ACL LOG: implement a few basic tests. + 1 file changed, 87 insertions(+) + +antirez in commit 82790e51: + ACL LOG: also log ACL errors in the scripting/MULTI ctx. + 2 files changed, 6 insertions(+), 2 deletions(-) + +antirez in commit 943008eb: + ACL LOG: implement LOG RESET. + 1 file changed, 6 insertions(+), 2 deletions(-) + +antirez in commit e271a611: + ACL LOG: group similar entries in a given time delta. + 1 file changed, 58 insertions(+), 3 deletions(-) + +antirez in commit f1974d5d: + ACL LOG: actually emit entries. + 3 files changed, 34 insertions(+), 5 deletions(-) + +antirez in commit d9b153c9: + ACL LOG: implement ACL LOG subcommadn skeleton. + 1 file changed, 37 insertions(+) + +antirez in commit 577fc438: + ACL LOG: data structures and initial functions. + 5 files changed, 54 insertions(+), 5 deletions(-) + +Leo Murillo in commit f7a94526: + Set ZSKIPLIST_MAXLEVEL to optimal value given 2^64 elements and p=0.25 + 1 file changed, 1 insertion(+), 1 deletion(-) + +WuYunlong in commit eecfa979: + Fix lua related memory leak. + 1 file changed, 1 insertion(+) + +WuYunlong in commit d2509811: + Add tcl regression test in scripting.tcl to reproduce memory leak. + 1 file changed, 5 insertions(+) + +Yossi Gottlieb in commit 29d4a150: + TLS: Fix missing initialization in redis-cli. + 1 file changed, 9 insertions(+) + +Oran Agra in commit ec0c61da: + fix uninitialized info_cb var in module.c + 1 file changed, 1 insertion(+) + +Guy Benoish in commit 6fe55c2f: + ld2string should fail if string contains \0 in the middle + 5 files changed, 20 insertions(+), 11 deletions(-) + +antirez in commit bbce3ba9: + Add more info in the unblockClientFromModule() function. + 1 file changed, 7 insertions(+), 1 deletion(-) + +Guy Benoish in commit 40295fb3: + Modules: Fix blocked-client-related memory leak + 3 files changed, 51 insertions(+), 6 deletions(-) + +antirez in commit 8e9d19bc: + Change error message for #6775. + 1 file changed, 2 insertions(+), 2 deletions(-) + +Vasyl Melnychuk in commit ba146d4c: + Make error when submitting command in incorrect context more explicit + 1 file changed, 4 insertions(+), 1 deletion(-) + +antirez in commit 721a39dd: + Document I/O threads in redis.conf. + 1 file changed, 46 insertions(+) + +antirez in commit 5be3a15a: + Setting N I/O threads should mean N-1 additional + 1 main thread. + 1 file changed, 25 insertions(+), 22 deletions(-) + +antirez in commit cbabf779: + Simplify #6379 changes. + 2 files changed, 4 insertions(+), 9 deletions(-) + +WuYunlong in commit 658749cc: + Free allocated sds in pfdebugCommand() to avoid memory leak. + 1 file changed, 1 insertion(+) + +WuYunlong in commit 47988c96: + Fix potential memory leak of clusterLoadConfig(). + 1 file changed, 20 insertions(+), 5 deletions(-) + +WuYunlong in commit cc90f79b: + Fix potential memory leak of rioWriteBulkStreamID(). + 1 file changed, 4 insertions(+), 1 deletion(-) + +antirez in commit ecd17e81: + Jump to right label on AOF parsing error. + 1 file changed, 6 insertions(+), 4 deletions(-) + +antirez in commit 1927932b: + Port PR #6110 to new connection object code. + 1 file changed, 2 insertions(+), 2 deletions(-) + +antirez in commit f2df5773: + A few comments about main thread serving I/O as well. + 1 file changed, 7 insertions(+), 1 deletion(-) + +zhaozhao.zz in commit b3ff8a4b: + Threaded IO: use main thread to handle read work + 1 file changed, 8 insertions(+), 1 deletion(-) + +zhaozhao.zz in commit b1f2c510: + Threaded IO: use main thread to handle write work + 1 file changed, 10 insertions(+), 2 deletions(-) + +ShooterIT in commit 7bbafc56: + Rename rdb asynchronously + 1 file changed, 7 insertions(+) + +Leo Murillo in commit c7f75266: + Fix bug on KEYS command where pattern starts with * followed by \x00 (null char). + 1 file changed, 1 insertion(+), 1 deletion(-) + +Jamie Scott in commit ed7ea13a: + Update to directive in redis.conf (missing s) + 1 file changed, 1 insertion(+), 1 deletion(-) + +antirez in commit 3be77623: + Free fakeclient argv on AOF error. + 1 file changed, 11 insertions(+), 3 deletions(-) + +antirez in commit 15f6b748: + Git ignore: ignore more files. + 1 file changed, 2 insertions(+) + +Guy Benoish in commit 1b5bf40c: + Blocking XREAD[GROUP] should always reply with valid data (or timeout) + 3 files changed, 44 insertions(+), 10 deletions(-) + +John Sully in commit 954c20ed: + Add support for incremental build with header files + 2 files changed, 6 insertions(+), 1 deletion(-) + +WuYunlong in commit 11c3afd7: + Fix petential cluster link error. + 1 file changed, 4 insertions(+) + +Yossi Gottlieb in commit b752e83d: + Add REDISMODULE_CTX_FLAGS_MULTI_DIRTY. + 2 files changed, 8 insertions(+) + +hwware in commit e16eb874: + typo fix in acl.c + 1 file changed, 2 insertions(+), 2 deletions(-) + +Itamar Haber in commit 35ea9d23: + Adjusts 'io_threads_num' max to 128 + 1 file changed, 1 insertion(+), 1 deletion(-) + +antirez in commit 38729126: + XCLAIM: Create the consumer only on successful claims. + 1 file changed, 4 insertions(+), 2 deletions(-) + +yz1509 in commit b9a15303: + avoid sentinel changes promoted_slave to be its own replica. + 1 file changed, 1 insertion(+), 1 deletion(-) + +antirez in commit 5e7e5e6b: + Fix active expire division by zero. + 1 file changed, 7 insertions(+), 4 deletions(-) + +antirez in commit e61dde88: + Fix duplicated CLIENT SETNAME reply. + 1 file changed, 1 deletion(-) + +Guy Benoish in commit cddf1da2: + Stream: Handle streamID-related edge cases + 4 files changed, 54 insertions(+), 4 deletions(-) + +Oran Agra in commit 52ea44e5: + config.c adjust config limits and mutable + 2 files changed, 7 insertions(+), 7 deletions(-) + +antirez in commit 0f28ea16: + Inline protocol: handle empty strings well. + 1 file changed, 2 insertions(+), 6 deletions(-) + +antirez in commit 00e5fefe: + Fix ip and missing mode in RM_GetClusterNodeInfo(). + 1 file changed, 5 insertions(+), 2 deletions(-) + ================================================================================ Redis 6.0 RC1 Released Thu Dec 19 09:58:24 CEST 2019 ================================================================================ diff --git a/src/version.h b/src/version.h index 1145da134..8db18a071 100644 --- a/src/version.h +++ b/src/version.h @@ -1 +1 @@ -#define REDIS_VERSION "5.9.101" +#define REDIS_VERSION "5.9.102" From b3e4aa6797f484ed08d3189af0dd7f143dbaf421 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 6 Mar 2020 11:09:52 +0100 Subject: [PATCH 125/225] Fix release notes spelling mistake. --- 00-RELEASENOTES | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/00-RELEASENOTES b/00-RELEASENOTES index ad2648bcb..9847d37dd 100644 --- a/00-RELEASENOTES +++ b/00-RELEASENOTES @@ -30,7 +30,7 @@ New features and improvements: * ACL LOG: log denied commands, keys accesses and authentications. * Client side caching redesigned. Now we use keys not caching slots. * Client side caching: Broadcasting mode implemented. -* Client side caching: OPTIN/OUTPUT modes implemented. +* Client side caching: OPTIN/OPTOUT modes implemented. * Remove RDB files used for replication in persistence-less instances (option). Fixes (only selected ones, see commits for all the fixes): From 70e0e49952ef40675b515a9483767be898640609 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 12 Mar 2020 12:59:44 +0100 Subject: [PATCH 126/225] ae.c: fix crash when resizing the event loop. See #6964. The root cause is that the event loop may be resized from an event callback itself, causing the event pointer to be invalid. --- src/ae.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/ae.c b/src/ae.c index d2248fe5c..1bf6cbfbf 100644 --- a/src/ae.c +++ b/src/ae.c @@ -464,6 +464,7 @@ int aeProcessEvents(aeEventLoop *eventLoop, int flags) if (!invert && fe->mask & mask & AE_READABLE) { fe->rfileProc(eventLoop,fd,fe->clientData,mask); fired++; + fe = &eventLoop->events[fd]; /* Refresh in case of resize. */ } /* Fire the writable event. */ @@ -476,8 +477,11 @@ int aeProcessEvents(aeEventLoop *eventLoop, int flags) /* If we have to invert the call, fire the readable event now * after the writable one. */ - if (invert && fe->mask & mask & AE_READABLE) { - if (!fired || fe->wfileProc != fe->rfileProc) { + if (invert) { + fe = &eventLoop->events[fd]; /* Refresh in case of resize. */ + if ((fe->mask & mask & AE_READABLE) && + (!fired || fe->wfileProc != fe->rfileProc)) + { fe->rfileProc(eventLoop,fd,fe->clientData,mask); fired++; } From 6173815408a379c5a060283c589e83c8f2f1134f Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Thu, 5 Mar 2020 16:55:14 +0200 Subject: [PATCH 127/225] fix for flaky psync2 test *** [err]: PSYNC2: total sum of full synchronizations is exactly 4 in tests/integration/psync2.tcl Expected 5 == 4 (context: type eval line 6 cmd {assert {$sum == 4}} proc ::test) issue was that sometime the test got an unexpected full sync since it tried to switch to the replica before it was in sync with it's master. --- tests/integration/psync2.tcl | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tests/integration/psync2.tcl b/tests/integration/psync2.tcl index d1212b640..333736ffa 100644 --- a/tests/integration/psync2.tcl +++ b/tests/integration/psync2.tcl @@ -114,6 +114,27 @@ start_server {} { } } + # wait for all the slaves to be in sync with the master + set master_ofs [status $R($master_id) master_repl_offset] + wait_for_condition 500 100 { + $master_ofs == [status $R(0) master_repl_offset] && + $master_ofs == [status $R(1) master_repl_offset] && + $master_ofs == [status $R(2) master_repl_offset] && + $master_ofs == [status $R(3) master_repl_offset] && + $master_ofs == [status $R(4) master_repl_offset] + } else { + if {$debug_msg} { + for {set j 0} {$j < 5} {incr j} { + puts "$j: sync_full: [status $R($j) sync_full]" + puts "$j: id1 : [status $R($j) master_replid]:[status $R($j) master_repl_offset]" + puts "$j: id2 : [status $R($j) master_replid2]:[status $R($j) second_repl_offset]" + puts "$j: backlog : firstbyte=[status $R($j) repl_backlog_first_byte_offset] len=[status $R($j) repl_backlog_histlen]" + puts "---" + } + } + fail "Slaves are not in sync with the master after too long time." + } + # Put down the old master so that it cannot generate more # replication stream, this way in the next master switch, the time at # which we move slaves away is not important, each will have full From 23d5e8b8e46c706bd5a93ed3b3e0782909e926b2 Mon Sep 17 00:00:00 2001 From: Johannes Truschnigg Date: Thu, 19 Dec 2019 21:47:24 +0100 Subject: [PATCH 128/225] Signal systemd readiness atfer Partial Resync "Partial Resynchronization" is a special variant of replication success that we have to tell systemd about if it is managing redis-server via a Type=Notify service unit. --- src/replication.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/replication.c b/src/replication.c index 429e1d8a8..31e14d7fe 100644 --- a/src/replication.c +++ b/src/replication.c @@ -2284,6 +2284,10 @@ void syncWithMaster(connection *conn) { if (psync_result == PSYNC_CONTINUE) { serverLog(LL_NOTICE, "MASTER <-> REPLICA sync: Master accepted a Partial Resynchronization."); + if (server.supervised_mode == SUPERVISED_SYSTEMD) { + redisCommunicateSystemd("STATUS=MASTER <-> REPLICA sync: Partial Resynchronization accepted. Ready to accept connections.\n"); + redisCommunicateSystemd("READY=1\n"); + } return; } From d28cbaf770734d5e839b42853ff96dc2d1dd6816 Mon Sep 17 00:00:00 2001 From: Jamie Scott Date: Tue, 3 Mar 2020 18:03:16 -0800 Subject: [PATCH 129/225] Update Redis.conf to improve TLS usability When using TLS with a Redis.conf file the line for TLS reading tls-cert-file redis.crt tls-key-file redis.key is interpreted as one complete directive. I am separating this on two separate lines to improve usability so users do not get the below error. ubuntu@ip-172-31-29-250:~/redis-6.0-rc1$ ./src/redis-server redis.conf *** FATAL CONFIG FILE ERROR *** Reading the configuration file, at line 145 >>> 'tls-cert-file redis.crt tls-key-file redis.key' wrong number of arguments ubuntu@ip-172-31-29-250:~/redis-6.0-rc1$ vi redis.conf ubuntu@ip-172-31-29-250:~/redis-6.0-rc1$ ./src/redis-server redis.conf 23085:C 04 Mar 2020 01:58:12.631 # oO0OoO0OoO0Oo Redis is starting oO0OoO0OoO0Oo 23085:C 04 Mar 2020 01:58:12.631 # Redis version=5.9.101, bits=64, commit=00000000, modified=0, pid=23085, just started 23085:C 04 Mar 2020 01:58:12.631 # Configuration loaded 23085:M 04 Mar 2020 01:58:12.632 * Increased maximum number of open files to 10032 (it was originally set to 1024). --- redis.conf | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/redis.conf b/redis.conf index 8609a9f57..6fa5e87c0 100644 --- a/redis.conf +++ b/redis.conf @@ -142,7 +142,8 @@ tcp-keepalive 300 # server to connected clients, masters or cluster peers. These files should be # PEM formatted. # -# tls-cert-file redis.crt tls-key-file redis.key +# tls-cert-file redis.crt +# tls-key-file redis.key # Configure a DH parameters file to enable Diffie-Hellman (DH) key exchange: # From e5a063bcb405399ea71ec4a6fdead61ad4b78bb2 Mon Sep 17 00:00:00 2001 From: Jamie Scott Date: Mon, 9 Mar 2020 12:53:44 -0700 Subject: [PATCH 130/225] Remove default guidance in Redis.conf Removing the default guidance in Redis.conf since this is not an available value. --- redis.conf | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/redis.conf b/redis.conf index 6fa5e87c0..c9d256bef 100644 --- a/redis.conf +++ b/redis.conf @@ -176,8 +176,7 @@ tcp-keepalive 300 # tls-cluster yes # Explicitly specify TLS versions to support. Allowed values are case insensitive -# and include "TLSv1", "TLSv1.1", "TLSv1.2", "TLSv1.3" (OpenSSL >= 1.1.1) or -# "default" which is currently >= TLSv1.1. +# and include "TLSv1", "TLSv1.1", "TLSv1.2", "TLSv1.3" (OpenSSL >= 1.1.1) # # tls-protocols TLSv1.2 From e1c29434b2f79425a1d006d7dd6737dd1fbbcc15 Mon Sep 17 00:00:00 2001 From: lifubang Date: Thu, 5 Mar 2020 18:13:43 +0800 Subject: [PATCH 131/225] update linenoise to https://github.com/antirez/linenoise/tree/fc9667a81d43911a6690fb1e68c16e6e3bb8df05 Signed-off-by: lifubang --- deps/linenoise/README.markdown | 25 ++++++++++++++++++++++++- deps/linenoise/example.c | 5 +++++ deps/linenoise/linenoise.c | 31 ++++++++++++++++++++++++++++--- deps/linenoise/linenoise.h | 2 ++ 4 files changed, 59 insertions(+), 4 deletions(-) diff --git a/deps/linenoise/README.markdown b/deps/linenoise/README.markdown index e01642cf8..1afea2ae6 100644 --- a/deps/linenoise/README.markdown +++ b/deps/linenoise/README.markdown @@ -21,7 +21,7 @@ So what usually happens is either: The result is a pollution of binaries without line editing support. -So I spent more or less two hours doing a reality check resulting in this little library: is it *really* needed for a line editing library to be 20k lines of code? Apparently not, it is possibe to get a very small, zero configuration, trivial to embed library, that solves the problem. Smaller programs will just include this, supporing line editing out of the box. Larger programs may use this little library or just checking with configure if readline/libedit is available and resorting to Linenoise if not. +So I spent more or less two hours doing a reality check resulting in this little library: is it *really* needed for a line editing library to be 20k lines of code? Apparently not, it is possibe to get a very small, zero configuration, trivial to embed library, that solves the problem. Smaller programs will just include this, supporting line editing out of the box. Larger programs may use this little library or just checking with configure if readline/libedit is available and resorting to Linenoise if not. ## Terminals, in 2010. @@ -126,6 +126,24 @@ Linenoise has direct support for persisting the history into an history file. The functions `linenoiseHistorySave` and `linenoiseHistoryLoad` do just that. Both functions return -1 on error and 0 on success. +## Mask mode + +Sometimes it is useful to allow the user to type passwords or other +secrets that should not be displayed. For such situations linenoise supports +a "mask mode" that will just replace the characters the user is typing +with `*` characters, like in the following example: + + $ ./linenoise_example + hello> get mykey + echo: 'get mykey' + hello> /mask + hello> ********* + +You can enable and disable mask mode using the following two functions: + + void linenoiseMaskModeEnable(void); + void linenoiseMaskModeDisable(void); + ## Completion Linenoise supports completion, which is the ability to complete the user @@ -222,3 +240,8 @@ Sometimes you may want to clear the screen as a result of something the user typed. You can do this by calling the following function: void linenoiseClearScreen(void); + +## Related projects + +* [Linenoise NG](https://github.com/arangodb/linenoise-ng) is a fork of Linenoise that aims to add more advanced features like UTF-8 support, Windows support and other features. Uses C++ instead of C as development language. +* [Linenoise-swift](https://github.com/andybest/linenoise-swift) is a reimplementation of Linenoise written in Swift. diff --git a/deps/linenoise/example.c b/deps/linenoise/example.c index 3a544d3c6..74358c323 100644 --- a/deps/linenoise/example.c +++ b/deps/linenoise/example.c @@ -55,6 +55,7 @@ int main(int argc, char **argv) { * * The typed string is returned as a malloc() allocated string by * linenoise, so the user needs to free() it. */ + while((line = linenoise("hello> ")) != NULL) { /* Do something with the string. */ if (line[0] != '\0' && line[0] != '/') { @@ -65,6 +66,10 @@ int main(int argc, char **argv) { /* The "/historylen" command will change the history len. */ int len = atoi(line+11); linenoiseHistorySetMaxLen(len); + } else if (!strncmp(line, "/mask", 5)) { + linenoiseMaskModeEnable(); + } else if (!strncmp(line, "/unmask", 7)) { + linenoiseMaskModeDisable(); } else if (line[0] == '/') { printf("Unreconized command: %s\n", line); } diff --git a/deps/linenoise/linenoise.c b/deps/linenoise/linenoise.c index fce14a7c5..01c3b9350 100644 --- a/deps/linenoise/linenoise.c +++ b/deps/linenoise/linenoise.c @@ -125,6 +125,7 @@ static linenoiseHintsCallback *hintsCallback = NULL; static linenoiseFreeHintsCallback *freeHintsCallback = NULL; static struct termios orig_termios; /* In order to restore at exit.*/ +static int maskmode = 0; /* Show "***" instead of input. For passwords. */ static int rawmode = 0; /* For atexit() function to check if restore is needed*/ static int mlmode = 0; /* Multi line mode. Default is single line. */ static int atexit_registered = 0; /* Register atexit just 1 time. */ @@ -197,6 +198,19 @@ FILE *lndebug_fp = NULL; /* ======================= Low level terminal handling ====================== */ +/* Enable "mask mode". When it is enabled, instead of the input that + * the user is typing, the terminal will just display a corresponding + * number of asterisks, like "****". This is useful for passwords and other + * secrets that should not be displayed. */ +void linenoiseMaskModeEnable(void) { + maskmode = 1; +} + +/* Disable mask mode. */ +void linenoiseMaskModeDisable(void) { + maskmode = 0; +} + /* Set if to use or not the multi line mode. */ void linenoiseSetMultiLine(int ml) { mlmode = ml; @@ -485,6 +499,8 @@ void refreshShowHints(struct abuf *ab, struct linenoiseState *l, int plen) { if (bold == 1 && color == -1) color = 37; if (color != -1 || bold != 0) snprintf(seq,64,"\033[%d;%d;49m",bold,color); + else + seq[0] = '\0'; abAppend(ab,seq,strlen(seq)); abAppend(ab,hint,hintlen); if (color != -1 || bold != 0) @@ -523,7 +539,11 @@ static void refreshSingleLine(struct linenoiseState *l) { abAppend(&ab,seq,strlen(seq)); /* Write the prompt and the current buffer content */ abAppend(&ab,l->prompt,strlen(l->prompt)); - abAppend(&ab,buf,len); + if (maskmode == 1) { + while (len--) abAppend(&ab,"*",1); + } else { + abAppend(&ab,buf,len); + } /* Show hits if any. */ refreshShowHints(&ab,l,plen); /* Erase to right */ @@ -577,7 +597,11 @@ static void refreshMultiLine(struct linenoiseState *l) { /* Write the prompt and the current buffer content */ abAppend(&ab,l->prompt,strlen(l->prompt)); - abAppend(&ab,l->buf,l->len); + if (maskmode == 1) { + for (uint i = 0; i < l->len; i++) abAppend(&ab,"*",1); + } else { + abAppend(&ab,l->buf,l->len); + } /* Show hits if any. */ refreshShowHints(&ab,l,plen); @@ -645,7 +669,8 @@ int linenoiseEditInsert(struct linenoiseState *l, char c) { if ((!mlmode && l->plen+l->len < l->cols && !hintsCallback)) { /* Avoid a full update of the line in the * trivial case. */ - if (write(l->ofd,&c,1) == -1) return -1; + char d = (maskmode==1) ? '*' : c; + if (write(l->ofd,&d,1) == -1) return -1; } else { refreshLine(l); } diff --git a/deps/linenoise/linenoise.h b/deps/linenoise/linenoise.h index ed20232c5..6dfee73bc 100644 --- a/deps/linenoise/linenoise.h +++ b/deps/linenoise/linenoise.h @@ -65,6 +65,8 @@ int linenoiseHistoryLoad(const char *filename); void linenoiseClearScreen(void); void linenoiseSetMultiLine(int ml); void linenoisePrintKeyCodes(void); +void linenoiseMaskModeEnable(void); +void linenoiseMaskModeDisable(void); #ifdef __cplusplus } From c0c67c9be30fa197ec86ba33915c6f6186a5f166 Mon Sep 17 00:00:00 2001 From: lifubang Date: Thu, 5 Mar 2020 18:17:32 +0800 Subject: [PATCH 132/225] add askpass mode Signed-off-by: lifubang --- src/redis-cli.c | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/src/redis-cli.c b/src/redis-cli.c index 54898f42e..b44db9a1e 100644 --- a/src/redis-cli.c +++ b/src/redis-cli.c @@ -229,6 +229,7 @@ static struct config { int hotkeys; int stdinarg; /* get last arg from stdin. (-x option) */ char *auth; + int askpass; char *user; int output; /* output mode, see OUTPUT_* defines */ sds mb_delim; @@ -1450,6 +1451,8 @@ static int parseOptions(int argc, char **argv) { config.dbnum = atoi(argv[++i]); } else if (!strcmp(argv[i], "--no-auth-warning")) { config.no_auth_warning = 1; + } else if (!strcmp(argv[i], "--askpass")) { + config.askpass = 1; } else if ((!strcmp(argv[i],"-a") || !strcmp(argv[i],"--pass")) && !lastarg) { @@ -1690,6 +1693,9 @@ static void usage(void) { " (if both are used, this argument takes predecence).\n" " --user Used to send ACL style 'AUTH username pass'. Needs -a.\n" " --pass Alias of -a for consistency with the new --user option.\n" +" --askpass Force user to input password with mask from STDIN.\n" +" If this argument is used, '-a' and " REDIS_CLI_AUTH_ENV "\n" +" environment variable will be ignored.\n" " -u Server URI.\n" " -r Execute specified command N times.\n" " -i When -r is used, waits seconds per command.\n" @@ -7858,6 +7864,13 @@ static void intrinsicLatencyMode(void) { } } +static sds askPassword() { + linenoiseMaskModeEnable(); + sds auth = linenoise("Please input password: "); + linenoiseMaskModeDisable(); + return auth; +} + /*------------------------------------------------------------------------------ * Program main() *--------------------------------------------------------------------------- */ @@ -7894,6 +7907,7 @@ int main(int argc, char **argv) { config.hotkeys = 0; config.stdinarg = 0; config.auth = NULL; + config.askpass = 0; config.user = NULL; config.eval = NULL; config.eval_ldb = 0; @@ -7935,6 +7949,10 @@ int main(int argc, char **argv) { parseEnv(); + if (config.askpass) { + config.auth = askPassword(); + } + #ifdef USE_OPENSSL if (config.tls) { ERR_load_crypto_strings(); @@ -8044,4 +8062,4 @@ int main(int argc, char **argv) { } else { return noninteractive(argc,convertToSds(argc,argv)); } -} +} \ No newline at end of file From 916dd79f190b51c78fdff7677a3a00d81e2088bf Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 12 Mar 2020 15:53:08 +0100 Subject: [PATCH 133/225] Update linenoise. --- deps/linenoise/linenoise.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/deps/linenoise/linenoise.c b/deps/linenoise/linenoise.c index 01c3b9350..cfe51e768 100644 --- a/deps/linenoise/linenoise.c +++ b/deps/linenoise/linenoise.c @@ -598,7 +598,8 @@ static void refreshMultiLine(struct linenoiseState *l) { /* Write the prompt and the current buffer content */ abAppend(&ab,l->prompt,strlen(l->prompt)); if (maskmode == 1) { - for (uint i = 0; i < l->len; i++) abAppend(&ab,"*",1); + unsigned int i; + for (i = 0; i < l->len; i++) abAppend(&ab,"*",1); } else { abAppend(&ab,l->buf,l->len); } From d387f67dcbb1e8c57aa8775ab9b53bbbeceda7c3 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 16 Mar 2020 15:59:29 +0100 Subject: [PATCH 134/225] Sentinel: implement auth-user directive for ACLs. --- src/sentinel.c | 45 ++++++++++++++++++++++++++++++++++++++------- 1 file changed, 38 insertions(+), 7 deletions(-) diff --git a/src/sentinel.c b/src/sentinel.c index 10c003d03..83d6c00bb 100644 --- a/src/sentinel.c +++ b/src/sentinel.c @@ -205,7 +205,8 @@ typedef struct sentinelRedisInstance { dict *slaves; /* Slaves for this master instance. */ unsigned int quorum;/* Number of sentinels that need to agree on failure. */ int parallel_syncs; /* How many slaves to reconfigure at same time. */ - char *auth_pass; /* Password to use for AUTH against master & slaves. */ + char *auth_pass; /* Password to use for AUTH against master & replica. */ + char *auth_user; /* Username for ACLs AUTH against master & replica. */ /* Slave specific. */ mstime_t master_link_down_time; /* Slave replication link down time. */ @@ -1231,6 +1232,7 @@ sentinelRedisInstance *createSentinelRedisInstance(char *name, int flags, char * SENTINEL_DEFAULT_DOWN_AFTER; ri->master_link_down_time = 0; ri->auth_pass = NULL; + ri->auth_user = NULL; ri->slave_priority = SENTINEL_DEFAULT_SLAVE_PRIORITY; ri->slave_reconf_sent_time = 0; ri->slave_master_host = NULL; @@ -1289,6 +1291,7 @@ void releaseSentinelRedisInstance(sentinelRedisInstance *ri) { sdsfree(ri->slave_master_host); sdsfree(ri->leader); sdsfree(ri->auth_pass); + sdsfree(ri->auth_user); sdsfree(ri->info); releaseSentinelAddr(ri->addr); dictRelease(ri->renamed_commands); @@ -1654,19 +1657,19 @@ char *sentinelHandleConfiguration(char **argv, int argc) { ri->failover_timeout = atoi(argv[2]); if (ri->failover_timeout <= 0) return "negative or zero time parameter."; - } else if (!strcasecmp(argv[0],"parallel-syncs") && argc == 3) { + } else if (!strcasecmp(argv[0],"parallel-syncs") && argc == 3) { /* parallel-syncs */ ri = sentinelGetMasterByName(argv[1]); if (!ri) return "No such master with specified name."; ri->parallel_syncs = atoi(argv[2]); - } else if (!strcasecmp(argv[0],"notification-script") && argc == 3) { + } else if (!strcasecmp(argv[0],"notification-script") && argc == 3) { /* notification-script */ ri = sentinelGetMasterByName(argv[1]); if (!ri) return "No such master with specified name."; if (access(argv[2],X_OK) == -1) return "Notification script seems non existing or non executable."; ri->notification_script = sdsnew(argv[2]); - } else if (!strcasecmp(argv[0],"client-reconfig-script") && argc == 3) { + } else if (!strcasecmp(argv[0],"client-reconfig-script") && argc == 3) { /* client-reconfig-script */ ri = sentinelGetMasterByName(argv[1]); if (!ri) return "No such master with specified name."; @@ -1674,11 +1677,16 @@ char *sentinelHandleConfiguration(char **argv, int argc) { return "Client reconfiguration script seems non existing or " "non executable."; ri->client_reconfig_script = sdsnew(argv[2]); - } else if (!strcasecmp(argv[0],"auth-pass") && argc == 3) { + } else if (!strcasecmp(argv[0],"auth-pass") && argc == 3) { /* auth-pass */ ri = sentinelGetMasterByName(argv[1]); if (!ri) return "No such master with specified name."; ri->auth_pass = sdsnew(argv[2]); + } else if (!strcasecmp(argv[0],"auth-user") && argc == 3) { + /* auth-user */ + ri = sentinelGetMasterByName(argv[1]); + if (!ri) return "No such master with specified name."; + ri->auth_user = sdsnew(argv[2]); } else if (!strcasecmp(argv[0],"current-epoch") && argc == 2) { /* current-epoch */ unsigned long long current_epoch = strtoull(argv[1],NULL,10); @@ -1836,7 +1844,7 @@ void rewriteConfigSentinelOption(struct rewriteConfigState *state) { rewriteConfigRewriteLine(state,"sentinel",line,1); } - /* sentinel auth-pass */ + /* sentinel auth-pass & auth-user */ if (master->auth_pass) { line = sdscatprintf(sdsempty(), "sentinel auth-pass %s %s", @@ -1844,6 +1852,13 @@ void rewriteConfigSentinelOption(struct rewriteConfigState *state) { rewriteConfigRewriteLine(state,"sentinel",line,1); } + if (master->auth_user) { + line = sdscatprintf(sdsempty(), + "sentinel auth-user %s %s", + master->name, master->auth_user); + rewriteConfigRewriteLine(state,"sentinel",line,1); + } + /* sentinel config-epoch */ line = sdscatprintf(sdsempty(), "sentinel config-epoch %s %llu", @@ -1968,19 +1983,29 @@ werr: * will disconnect and reconnect the link and so forth. */ void sentinelSendAuthIfNeeded(sentinelRedisInstance *ri, redisAsyncContext *c) { char *auth_pass = NULL; + char *auth_user = NULL; if (ri->flags & SRI_MASTER) { auth_pass = ri->auth_pass; + auth_user = ri->auth_user; } else if (ri->flags & SRI_SLAVE) { auth_pass = ri->master->auth_pass; + auth_user = ri->master->auth_user; } else if (ri->flags & SRI_SENTINEL) { auth_pass = ACLDefaultUserFirstPassword(); + auth_user = NULL; } - if (auth_pass) { + if (auth_pass && auth_user == NULL) { if (redisAsyncCommand(c, sentinelDiscardReplyCallback, ri, "%s %s", sentinelInstanceMapCommand(ri,"AUTH"), auth_pass) == C_OK) ri->link->pending_commands++; + } else if (auth_pass && auth_user) { + /* If we also have an username, use the ACL-style AUTH command + * with two arguments, username and password. */ + if (redisAsyncCommand(c, sentinelDiscardReplyCallback, ri, "%s %s %s", + sentinelInstanceMapCommand(ri,"AUTH"), + auth_user, auth_pass) == C_OK) ri->link->pending_commands++; } } @@ -3522,6 +3547,12 @@ void sentinelSetCommand(client *c) { sdsfree(ri->auth_pass); ri->auth_pass = strlen(value) ? sdsnew(value) : NULL; changes++; + } else if (!strcasecmp(option,"auth-user") && moreargs > 0) { + /* auth-user */ + char *value = c->argv[++j]->ptr; + sdsfree(ri->auth_user); + ri->auth_user = strlen(value) ? sdsnew(value) : NULL; + changes++; } else if (!strcasecmp(option,"quorum") && moreargs > 0) { /* quorum */ robj *o = c->argv[++j]; From 9c2e42ddfc55505390d81b8110158ef9aba7b127 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 16 Mar 2020 16:56:50 +0100 Subject: [PATCH 135/225] ACL: Make Redis 6 more backward compatible with requirepass. Note that this as a side effect fixes Sentinel "requirepass" mode. --- src/acl.c | 11 +---------- src/config.c | 16 ++++++++++++---- src/sentinel.c | 2 +- src/server.h | 3 +++ 4 files changed, 17 insertions(+), 15 deletions(-) diff --git a/src/acl.c b/src/acl.c index efe6b96ad..27f4bdb84 100644 --- a/src/acl.c +++ b/src/acl.c @@ -899,16 +899,6 @@ char *ACLSetUserStringError(void) { return errmsg; } -/* Return the first password of the default user or NULL. - * This function is needed for backward compatibility with the old - * directive "requirepass" when Redis supported a single global - * password. */ -sds ACLDefaultUserFirstPassword(void) { - if (listLength(DefaultUser->passwords) == 0) return NULL; - listNode *first = listFirst(DefaultUser->passwords); - return listNodeValue(first); -} - /* Initialize the default user, that will always exist for all the process * lifetime. */ void ACLInitDefaultUser(void) { @@ -925,6 +915,7 @@ void ACLInit(void) { UsersToLoad = listCreate(); ACLLog = listCreate(); ACLInitDefaultUser(); + server.requirepass = NULL; /* Only used for backward compatibility. */ } /* Check the username and password pair and return C_OK if they are valid, diff --git a/src/config.c b/src/config.c index 211b6d003..7c87ebe6e 100644 --- a/src/config.c +++ b/src/config.c @@ -411,11 +411,15 @@ void loadServerConfigFromString(char *config) { goto loaderr; } /* The old "requirepass" directive just translates to setting - * a password to the default user. */ + * a password to the default user. The only thing we do + * additionally is to remember the cleartext password in this + * case, for backward compatibility with Redis <= 5. */ ACLSetUser(DefaultUser,"resetpass",-1); sds aclop = sdscatprintf(sdsempty(),">%s",argv[1]); ACLSetUser(DefaultUser,aclop,sdslen(aclop)); sdsfree(aclop); + sdsfree(server.requirepass); + server.requirepass = sdsnew(argv[1]); } else if (!strcasecmp(argv[0],"list-max-ziplist-entries") && argc == 2){ /* DEAD OPTION */ } else if (!strcasecmp(argv[0],"list-max-ziplist-value") && argc == 2) { @@ -623,11 +627,15 @@ void configSetCommand(client *c) { config_set_special_field("requirepass") { if (sdslen(o->ptr) > CONFIG_AUTHPASS_MAX_LEN) goto badfmt; /* The old "requirepass" directive just translates to setting - * a password to the default user. */ + * a password to the default user. The only thing we do + * additionally is to remember the cleartext password in this + * case, for backward compatibility with Redis <= 5. */ ACLSetUser(DefaultUser,"resetpass",-1); sds aclop = sdscatprintf(sdsempty(),">%s",(char*)o->ptr); ACLSetUser(DefaultUser,aclop,sdslen(aclop)); sdsfree(aclop); + sdsfree(server.requirepass); + server.requirepass = sdsnew(o->ptr); } config_set_special_field("save") { int vlen, j; sds *v = sdssplitlen(o->ptr,sdslen(o->ptr)," ",1,&vlen); @@ -899,7 +907,7 @@ void configGetCommand(client *c) { } if (stringmatch(pattern,"requirepass",1)) { addReplyBulkCString(c,"requirepass"); - sds password = ACLDefaultUserFirstPassword(); + sds password = server.requirepass; if (password) { addReplyBulkCBuffer(c,password,sdslen(password)); } else { @@ -1341,7 +1349,7 @@ void rewriteConfigBindOption(struct rewriteConfigState *state) { void rewriteConfigRequirepassOption(struct rewriteConfigState *state, char *option) { int force = 1; sds line; - sds password = ACLDefaultUserFirstPassword(); + sds password = server.requirepass; /* If there is no password set, we don't want the requirepass option * to be present in the configuration at all. */ diff --git a/src/sentinel.c b/src/sentinel.c index 83d6c00bb..d091bf230 100644 --- a/src/sentinel.c +++ b/src/sentinel.c @@ -1992,7 +1992,7 @@ void sentinelSendAuthIfNeeded(sentinelRedisInstance *ri, redisAsyncContext *c) { auth_pass = ri->master->auth_pass; auth_user = ri->master->auth_user; } else if (ri->flags & SRI_SENTINEL) { - auth_pass = ACLDefaultUserFirstPassword(); + auth_pass = server.requirepass; auth_user = NULL; } diff --git a/src/server.h b/src/server.h index 3c19a17ea..fa6770dfa 100644 --- a/src/server.h +++ b/src/server.h @@ -1395,6 +1395,9 @@ struct redisServer { /* ACLs */ char *acl_filename; /* ACL Users file. NULL if not configured. */ unsigned long acllog_max_len; /* Maximum length of the ACL LOG list. */ + sds requirepass; /* Remember the cleartext password set with the + old "requirepass" directive for backward + compatibility with Redis <= 5. */ /* Assert & bug reporting */ const char *assert_failed; const char *assert_file; From cbbf9b3931c517aa59fc9ede9e79184f0c0cc69e Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 16 Mar 2020 17:11:43 +0100 Subject: [PATCH 136/225] Sentinel: document auth-user directive. --- sentinel.conf | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/sentinel.conf b/sentinel.conf index 796f45088..4ca5e5f8f 100644 --- a/sentinel.conf +++ b/sentinel.conf @@ -102,6 +102,18 @@ sentinel monitor mymaster 127.0.0.1 6379 2 # # sentinel auth-pass mymaster MySUPER--secret-0123passw0rd +# sentinel auth-user +# +# This is useful in order to authenticate to instances having ACL capabilities, +# that is, running Redis 6.0 or greater. When just auth-pass is provided the +# Sentinel instance will authenticate to Redis using the old "AUTH " +# method. When also an username is provided, it will use "AUTH ". +# In the Redis servers side, the ACL to provide just minimal access to +# Sentinel instances, should be configured along the following lines: +# +# user sentinel-user >somepassword +client +subscribe +publish \ +# +ping +info +multi +slaveof +config +client +exec on + # sentinel down-after-milliseconds # # Number of milliseconds the master (or any attached replica or sentinel) should From 34ea2f4e1a7df2f1ae1404e7194c55b253323c87 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 20 Mar 2020 12:45:48 +0100 Subject: [PATCH 137/225] ACL: default user off should not allow automatic authentication. This fixes issue #7011. --- src/networking.c | 3 ++- src/server.c | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/networking.c b/src/networking.c index 0690bbdf6..69d59a59b 100644 --- a/src/networking.c +++ b/src/networking.c @@ -124,7 +124,8 @@ client *createClient(connection *conn) { c->ctime = c->lastinteraction = server.unixtime; /* If the default user does not require authentication, the user is * directly authenticated. */ - c->authenticated = (c->user->flags & USER_FLAG_NOPASS) != 0; + c->authenticated = (c->user->flags & USER_FLAG_NOPASS) && + !(c->user->flags & USER_FLAG_DISABLED); c->replstate = REPL_STATE_NONE; c->repl_put_online_on_ack = 0; c->reploff = 0; diff --git a/src/server.c b/src/server.c index f702da94a..612805ce5 100644 --- a/src/server.c +++ b/src/server.c @@ -3380,7 +3380,7 @@ int processCommand(client *c) { /* Check if the user is authenticated. This check is skipped in case * the default user is flagged as "nopass" and is active. */ int auth_required = (!(DefaultUser->flags & USER_FLAG_NOPASS) || - DefaultUser->flags & USER_FLAG_DISABLED) && + (DefaultUser->flags & USER_FLAG_DISABLED)) && !c->authenticated; if (auth_required) { /* AUTH and HELLO and no auth modules are valid even in From bfb18e55199788c8851ba77cdcbf1d53af98bbf4 Mon Sep 17 00:00:00 2001 From: "bodong.ybd" Date: Thu, 12 Mar 2020 11:12:37 +0800 Subject: [PATCH 138/225] Remove duplicate obj files in Makefile --- src/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Makefile b/src/Makefile index 00b623a4b..bbfb06440 100644 --- a/src/Makefile +++ b/src/Makefile @@ -208,9 +208,9 @@ REDIS_SERVER_NAME=redis-server REDIS_SENTINEL_NAME=redis-sentinel REDIS_SERVER_OBJ=adlist.o quicklist.o ae.o anet.o dict.o server.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o cluster.o crc16.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o memtest.o crc64.o bitops.o sentinel.o notify.o setproctitle.o blocked.o hyperloglog.o latency.o sparkline.o redis-check-rdb.o redis-check-aof.o geo.o lazyfree.o module.o evict.o expire.o geohash.o geohash_helper.o childinfo.o defrag.o siphash.o rax.o t_stream.o listpack.o localtime.o lolwut.o lolwut5.o lolwut6.o acl.o gopher.o tracking.o connection.o tls.o sha256.o REDIS_CLI_NAME=redis-cli -REDIS_CLI_OBJ=anet.o adlist.o dict.o redis-cli.o zmalloc.o release.o anet.o ae.o crc64.o siphash.o crc16.o +REDIS_CLI_OBJ=anet.o adlist.o dict.o redis-cli.o zmalloc.o release.o ae.o crc64.o siphash.o crc16.o REDIS_BENCHMARK_NAME=redis-benchmark -REDIS_BENCHMARK_OBJ=ae.o anet.o redis-benchmark.o adlist.o dict.o zmalloc.o siphash.o redis-benchmark.o +REDIS_BENCHMARK_OBJ=ae.o anet.o redis-benchmark.o adlist.o dict.o zmalloc.o siphash.o REDIS_CHECK_RDB_NAME=redis-check-rdb REDIS_CHECK_AOF_NAME=redis-check-aof From 61b98f32a298ed1b971dd7ae5d32db9b9a8e5cc9 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 20 Mar 2020 12:52:06 +0100 Subject: [PATCH 139/225] Regression test for #7011. --- tests/unit/acl.tcl | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/unit/acl.tcl b/tests/unit/acl.tcl index fc1664a75..85c9b81a9 100644 --- a/tests/unit/acl.tcl +++ b/tests/unit/acl.tcl @@ -248,4 +248,11 @@ start_server {tags {"acl"}} { r AUTH default "" assert {[llength [r ACL LOG]] == 5} } + + test {When default user is off, new connections are not authenticated} { + r ACL setuser default off + catch {set rd1 [redis_deferring_client]} e + r ACL setuser default on + set e + } {*NOAUTH*} } From dc8885a1ca025a2a10064d14d242c6125bf96342 Mon Sep 17 00:00:00 2001 From: Itamar Haber Date: Wed, 11 Mar 2020 18:43:03 +0200 Subject: [PATCH 140/225] Adds keyspace notifications to migrate and restore --- src/cluster.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/cluster.c b/src/cluster.c index c05e46f76..5f63d2b8f 100644 --- a/src/cluster.c +++ b/src/cluster.c @@ -4981,6 +4981,7 @@ void restoreCommand(client *c) { } objectSetLRUOrLFU(obj,lfu_freq,lru_idle,lru_clock,1000); signalModifiedKey(c->db,c->argv[1]); + notifyKeyspaceEvent(NOTIFY_GENERIC,"restore",c->argv[1],c->db->id); addReply(c,shared.ok); server.dirty++; } @@ -5327,6 +5328,7 @@ try_again: /* No COPY option: remove the local key, signal the change. */ dbDelete(c->db,kv[j]); signalModifiedKey(c->db,kv[j]); + notifyKeyspaceEvent(NOTIFY_GENERIC,"del",kv[j],c->db->id); server.dirty++; /* Populate the argument vector to replace the old one. */ @@ -5489,7 +5491,7 @@ void readwriteCommand(client *c) { * already "down" but it is fragile to rely on the update of the global state, * so we also handle it here. * - * CLUSTER_REDIR_DOWN_STATE and CLUSTER_REDIR_DOWN_RO_STATE if the cluster is + * CLUSTER_REDIR_DOWN_STATE and CLUSTER_REDIR_DOWN_RO_STATE if the cluster is * down but the user attempts to execute a command that addresses one or more keys. */ clusterNode *getNodeByQuery(client *c, struct redisCommand *cmd, robj **argv, int argc, int *hashslot, int *error_code) { clusterNode *n = NULL; From bdb338cf77c1eb59363d165eb46bb7259367642c Mon Sep 17 00:00:00 2001 From: antirez Date: Sun, 15 Mar 2020 16:10:37 +0100 Subject: [PATCH 141/225] Aesthetic changes in PR #6989. --- src/networking.c | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/networking.c b/src/networking.c index 261c9b24a..c7dd2f23c 100644 --- a/src/networking.c +++ b/src/networking.c @@ -36,7 +36,7 @@ static void setProtocolError(const char *errstr, client *c); int postponeClientRead(client *c); -int process_while_blocked; +int ProcessingEventsWhileBlocked = 0; /* See processEventsWhileBlocked(). */ /* Return the size consumed from the allocator, for the specified SDS string, * including internal fragmentation. This function is used in order to compute @@ -2739,7 +2739,12 @@ int clientsArePaused(void) { int processEventsWhileBlocked(void) { int iterations = 4; /* See the function top-comment. */ int count = 0; - process_while_blocked = 1; + + /* Note: when we are processing events while blocked (for instance during + * busy Lua scripts), we set a global flag. When such flag is set, we + * avoid handling the read part of clients using threaded I/O. + * See https://github.com/antirez/redis/issues/6988 for more info. */ + ProcessingEventsWhileBlocked = 1; while (iterations--) { int events = 0; events += aeProcessEvents(server.el, AE_FILE_EVENTS|AE_DONT_WAIT); @@ -2747,7 +2752,7 @@ int processEventsWhileBlocked(void) { if (!events) break; count += events; } - process_while_blocked = 0; + ProcessingEventsWhileBlocked = 0; return count; } @@ -2819,7 +2824,6 @@ void *IOThreadMain(void *myid) { /* Initialize the data structures needed for threaded I/O. */ void initThreadedIO(void) { io_threads_active = 0; /* We start with threads not active. */ - process_while_blocked = 0; /* Don't spawn any thread if the user selected a single thread: * we'll handle I/O directly from the main thread. */ @@ -2974,7 +2978,7 @@ int handleClientsWithPendingWritesUsingThreads(void) { int postponeClientRead(client *c) { if (io_threads_active && server.io_threads_do_reads && - !process_while_blocked && + !ProcessingEventsWhileBlocked && !(c->flags & (CLIENT_MASTER|CLIENT_SLAVE|CLIENT_PENDING_READ))) { c->flags |= CLIENT_PENDING_READ; From 299f1d0258e0d489b9c7438f840e14b8789f37ce Mon Sep 17 00:00:00 2001 From: WuYunlong Date: Wed, 18 Mar 2020 16:17:46 +0800 Subject: [PATCH 142/225] Add 14-consistency-check.tcl to prove there is a data consistency issue. --- tests/cluster/tests/14-consistency-check.tcl | 87 ++++++++++++++++++++ 1 file changed, 87 insertions(+) create mode 100644 tests/cluster/tests/14-consistency-check.tcl diff --git a/tests/cluster/tests/14-consistency-check.tcl b/tests/cluster/tests/14-consistency-check.tcl new file mode 100644 index 000000000..a43725ebc --- /dev/null +++ b/tests/cluster/tests/14-consistency-check.tcl @@ -0,0 +1,87 @@ +source "../tests/includes/init-tests.tcl" + +test "Create a 5 nodes cluster" { + create_cluster 5 5 +} + +test "Cluster should start ok" { + assert_cluster_state ok +} + +test "Cluster is writable" { + cluster_write_test 0 +} + +proc find_non_empty_master {} { + set master_id_no {} + foreach_redis_id id { + if {[RI $id role] eq {master} && [R $id dbsize] > 0} { + set master_id_no $id + } + } + return $master_id_no +} + +proc get_one_of_my_replica {id} { + set replica_port [lindex [lindex [lindex [R $id role] 2] 0] 1] + set replica_id_num [get_instance_id_by_port redis $replica_port] + return $replica_id_num +} + +proc cluster_write_keys_with_expire {id ttl} { + set prefix [randstring 20 20 alpha] + set port [get_instance_attrib redis $id port] + set cluster [redis_cluster 127.0.0.1:$port] + for {set j 100} {$j < 200} {incr j} { + $cluster setex key_expire.$j $ttl $prefix.$j + } + $cluster close +} + +proc test_slave_load_expired_keys {aof} { + test "Slave expired keys is loaded when restarted: appendonly=$aof" { + set master_id [find_non_empty_master] + set replica_id [get_one_of_my_replica $master_id] + + set master_dbsize [R $master_id dbsize] + set slave_dbsize [R $replica_id dbsize] + assert_equal $master_dbsize $slave_dbsize + + set data_ttl 5 + cluster_write_keys_with_expire $master_id $data_ttl + after 100 + set replica_dbsize_1 [R $replica_id dbsize] + assert {$replica_dbsize_1 > $slave_dbsize} + + R $replica_id config set appendonly $aof + R $replica_id config rewrite + + set start_time [clock seconds] + set end_time [expr $start_time+$data_ttl+2] + R $replica_id save + set replica_dbsize_2 [R $replica_id dbsize] + assert {$replica_dbsize_2 > $slave_dbsize} + kill_instance redis $replica_id + + set master_port [get_instance_attrib redis $master_id port] + exec ../../../src/redis-cli -h 127.0.0.1 -p $master_port debug sleep [expr $data_ttl+3] > /dev/null & + + while {[clock seconds] <= $end_time} { + #wait for $data_ttl seconds + } + restart_instance redis $replica_id + + wait_for_condition 200 50 { + [R $replica_id ping] eq {PONG} + } else { + fail "replica #$replica_id not started" + } + + set replica_dbsize_3 [R $replica_id dbsize] + assert {$replica_dbsize_3 > $slave_dbsize} + } +} + +test_slave_load_expired_keys no +after 5000 +test_slave_load_expired_keys yes From da14982d1ed8a1f2719f9206be8b8105db22745b Mon Sep 17 00:00:00 2001 From: guodongxiaren <879231132@qq.com> Date: Sat, 7 Mar 2020 19:38:27 +0800 Subject: [PATCH 143/225] string literal should be const char* --- src/asciilogo.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/asciilogo.h b/src/asciilogo.h index 83c538b54..044ca0c55 100644 --- a/src/asciilogo.h +++ b/src/asciilogo.h @@ -27,7 +27,7 @@ * POSSIBILITY OF SUCH DAMAGE. */ -char *ascii_logo = +const char *ascii_logo = " _._ \n" " _.-``__ ''-._ \n" " _.-`` `. `_. ''-._ Redis %s (%s/%d) %s bit\n" From da8c7c49bf534f6c75c2cb2724a3060e8b6bbab2 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 16 Mar 2020 13:48:29 +0100 Subject: [PATCH 144/225] Example sentinel conf: document requirepass. --- sentinel.conf | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/sentinel.conf b/sentinel.conf index bc9a705ac..796f45088 100644 --- a/sentinel.conf +++ b/sentinel.conf @@ -112,6 +112,14 @@ sentinel monitor mymaster 127.0.0.1 6379 2 # Default is 30 seconds. sentinel down-after-milliseconds mymaster 30000 +# requirepass +# +# You can configure Sentinel itself to require a password, however when doing +# so Sentinel will try to authenticate with the same password to all the +# other Sentinels. So you need to configure all your Sentinels in a given +# group with the same "requirepass" password. Check the following documentation +# for more info: https://redis.io/topics/sentinel + # sentinel parallel-syncs # # How many replicas we can reconfigure to point to the new replica simultaneously From 0578157d569e673d5c7728ea7941a4e0f24a4d48 Mon Sep 17 00:00:00 2001 From: WuYunlong Date: Wed, 18 Mar 2020 16:20:10 +0800 Subject: [PATCH 145/225] Fix master replica inconsistency for upgrading scenario. Before this commit, when upgrading a replica, expired keys will not be loaded, thus causing replica having less keys in db. To this point, master and replica's keys is logically consistent. However, before the keys in master and replica are physically consistent, that is, they have the same dbsize, if master got a problem and the replica got promoted and becomes new master of that partition, and master updates a key which does not exist on master, but physically exists on the old master(new replica), the old master would refuse to update the key, thus causing master and replica data inconsistent. How could this happen? That's all because of the wrong judgement of roles while starting up the server. We can not use server.masterhost to judge if the server is master or replica, since it fails in cluster mode. When we start the server, we load rdb and do want to load expired keys, and do not want to have the ability to active expire keys, if it is a replica. --- src/rdb.c | 2 +- src/server.c | 7 ++++++- src/server.h | 2 ++ 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/rdb.c b/src/rdb.c index cbcea96c6..5d34f5a32 100644 --- a/src/rdb.c +++ b/src/rdb.c @@ -2231,7 +2231,7 @@ int rdbLoadRio(rio *rdb, int rdbflags, rdbSaveInfo *rsi) { * received from the master. In the latter case, the master is * responsible for key expiry. If we would expire keys here, the * snapshot taken by the master may not be reflected on the slave. */ - if (server.masterhost == NULL && !(rdbflags&RDBFLAGS_AOF_PREAMBLE) && expiretime != -1 && expiretime < now) { + if (iAmMaster() && !(rdbflags&RDBFLAGS_AOF_PREAMBLE) && expiretime != -1 && expiretime < now) { decrRefCount(key); decrRefCount(val); } else { diff --git a/src/server.c b/src/server.c index 612805ce5..4b010b870 100644 --- a/src/server.c +++ b/src/server.c @@ -1691,7 +1691,7 @@ void databasesCron(void) { /* Expire keys by random sampling. Not required for slaves * as master will synthesize DELs for us. */ if (server.active_expire_enabled) { - if (server.masterhost == NULL) { + if (iAmMaster()) { activeExpireCycle(ACTIVE_EXPIRE_CYCLE_SLOW); } else { expireSlaveKeys(); @@ -4863,6 +4863,11 @@ int redisIsSupervised(int mode) { return 0; } +int iAmMaster(void) { + return ((!server.cluster_enabled && server.masterhost == NULL) || + (server.cluster_enabled && nodeIsMaster(server.cluster->myself))); +} + int main(int argc, char **argv) { struct timeval tv; diff --git a/src/server.h b/src/server.h index fa6770dfa..fdfe5b8ea 100644 --- a/src/server.h +++ b/src/server.h @@ -2393,4 +2393,6 @@ int tlsConfigure(redisTLSContextConfig *ctx_config); #define redisDebugMark() \ printf("-- MARK %s:%d --\n", __FILE__, __LINE__) +int iAmMaster(void); + #endif From 0e5820d893467b2607515e3ab21c824b4ce3fa8c Mon Sep 17 00:00:00 2001 From: fengpf <18221167541@163.com> Date: Thu, 12 Mar 2020 20:44:32 +0800 Subject: [PATCH 146/225] fix comments in latency.c --- .gitignore | 1 + src/latency.c | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index de626d61b..e445fd201 100644 --- a/.gitignore +++ b/.gitignore @@ -32,3 +32,4 @@ deps/lua/src/liblua.a *.dSYM Makefile.dep .vscode/* +.idea/* diff --git a/src/latency.c b/src/latency.c index 74ced72a5..9a291ac9b 100644 --- a/src/latency.c +++ b/src/latency.c @@ -85,7 +85,7 @@ int THPGetAnonHugePagesSize(void) { /* ---------------------------- Latency API --------------------------------- */ /* Latency monitor initialization. We just need to create the dictionary - * of time series, each time serie is craeted on demand in order to avoid + * of time series, each time serie is created on demand in order to avoid * having a fixed list to maintain. */ void latencyMonitorInit(void) { server.latency_events = dictCreate(&latencyTimeSeriesDictType,NULL); From 5d4c4df3efa4009035ba28329195d4e1378f97ba Mon Sep 17 00:00:00 2001 From: chendianqiang Date: Sat, 22 Feb 2020 15:03:01 +0800 Subject: [PATCH 147/225] use correct list for moduleUnregisterUsedAPI --- src/module.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/module.c b/src/module.c index bbd54082c..74da6c24d 100644 --- a/src/module.c +++ b/src/module.c @@ -6234,7 +6234,7 @@ int moduleUnregisterUsedAPI(RedisModule *module) { RedisModule *used = ln->value; listNode *ln = listSearchKey(used->usedby,module); if (ln) { - listDelNode(module->using,ln); + listDelNode(used->usedby,ln); count++; } } From 9cc7038e5473cdab8232e58002e9741c3637366f Mon Sep 17 00:00:00 2001 From: "zhaozhao.zz" Date: Mon, 16 Mar 2020 11:20:48 +0800 Subject: [PATCH 148/225] Threaded IO: handle pending reads clients ASAP after event loop --- src/server.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/server.c b/src/server.c index a6d4b357e..f702da94a 100644 --- a/src/server.c +++ b/src/server.c @@ -2088,6 +2088,9 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) { void beforeSleep(struct aeEventLoop *eventLoop) { UNUSED(eventLoop); + /* We should handle pending reads clients ASAP after event loop. */ + handleClientsWithPendingReadsUsingThreads(); + /* Handle TLS pending data. (must be done before flushAppendOnlyFile) */ tlsProcessPendingData(); /* If tls still has pending unread data don't sleep at all. */ @@ -2157,7 +2160,6 @@ void beforeSleep(struct aeEventLoop *eventLoop) { void afterSleep(struct aeEventLoop *eventLoop) { UNUSED(eventLoop); if (moduleCount()) moduleAcquireGIL(); - handleClientsWithPendingReadsUsingThreads(); } /* =========================== Server initialization ======================== */ From 04c53fa145273d531229d0324fef22494e17868a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EB=B0=95=EC=8A=B9=ED=98=84?= Date: Wed, 18 Mar 2020 14:40:50 +0900 Subject: [PATCH 149/225] Update redis.conf --- redis.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/redis.conf b/redis.conf index c9d256bef..7c55a3ab0 100644 --- a/redis.conf +++ b/redis.conf @@ -1628,7 +1628,7 @@ hz 10 # offers, and enables by default, the ability to use an adaptive HZ value # which will temporary raise when there are many connected clients. # -# When dynamic HZ is enabled, the actual configured HZ will be used as +# When dynamic HZ is enabled, the actual configured HZ will be used # as a baseline, but multiples of the configured HZ value will be actually # used as needed once more clients are connected. In this way an idle # instance will use very little CPU time while a busy instance will be From 76d57161d960f27ac92765118e3b80fef8847b8f Mon Sep 17 00:00:00 2001 From: "bodong.ybd" Date: Wed, 11 Mar 2020 20:55:51 +0800 Subject: [PATCH 150/225] Fix bug of tcl test using external server --- tests/support/server.tcl | 7 +++++-- tests/test_helper.tcl | 3 +++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/support/server.tcl b/tests/support/server.tcl index d086366dc..400017c5f 100644 --- a/tests/support/server.tcl +++ b/tests/support/server.tcl @@ -159,9 +159,12 @@ proc start_server {options {code undefined}} { if {$::external} { if {[llength $::servers] == 0} { set srv {} + # In test_server_main(tests/test_helper.tcl:215~218), increase the value of start_port + # and assign it to ::port through the `--port` option, so we need to reduce it. + set baseport [expr {$::port-100}] dict set srv "host" $::host - dict set srv "port" $::port - set client [redis $::host $::port 0 $::tls] + dict set srv "port" $baseport + set client [redis $::host $baseport 0 $::tls] dict set srv "client" $client $client select 9 diff --git a/tests/test_helper.tcl b/tests/test_helper.tcl index fa5579669..4dbead193 100644 --- a/tests/test_helper.tcl +++ b/tests/test_helper.tcl @@ -505,6 +505,9 @@ for {set j 0} {$j < [llength $argv]} {incr j} { } elseif {$opt eq {--host}} { set ::external 1 set ::host $arg + # If we use an external server, we can only set numclients to 1, + # otherwise the port will be miscalculated. + set ::numclients 1 incr j } elseif {$opt eq {--port}} { set ::port $arg From e628f94436f2ffb38dd6305f18956f309838a8d6 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 13 Mar 2020 16:21:55 +0100 Subject: [PATCH 151/225] Restore newline at the end of redis-cli.c --- src/redis-cli.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/redis-cli.c b/src/redis-cli.c index b44db9a1e..7ad80c0a1 100644 --- a/src/redis-cli.c +++ b/src/redis-cli.c @@ -8062,4 +8062,5 @@ int main(int argc, char **argv) { } else { return noninteractive(argc,convertToSds(argc,argv)); } -} \ No newline at end of file +} + From b3e0305448422239ba3cb8d125681bccdb809cd3 Mon Sep 17 00:00:00 2001 From: "zhaozhao.zz" Date: Sun, 15 Mar 2020 21:49:10 +0800 Subject: [PATCH 152/225] Threaded IO: bugfix #6988 process events while blocked --- src/networking.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/networking.c b/src/networking.c index 4c394af70..261c9b24a 100644 --- a/src/networking.c +++ b/src/networking.c @@ -36,6 +36,7 @@ static void setProtocolError(const char *errstr, client *c); int postponeClientRead(client *c); +int process_while_blocked; /* Return the size consumed from the allocator, for the specified SDS string, * including internal fragmentation. This function is used in order to compute @@ -2738,6 +2739,7 @@ int clientsArePaused(void) { int processEventsWhileBlocked(void) { int iterations = 4; /* See the function top-comment. */ int count = 0; + process_while_blocked = 1; while (iterations--) { int events = 0; events += aeProcessEvents(server.el, AE_FILE_EVENTS|AE_DONT_WAIT); @@ -2745,6 +2747,7 @@ int processEventsWhileBlocked(void) { if (!events) break; count += events; } + process_while_blocked = 0; return count; } @@ -2816,6 +2819,7 @@ void *IOThreadMain(void *myid) { /* Initialize the data structures needed for threaded I/O. */ void initThreadedIO(void) { io_threads_active = 0; /* We start with threads not active. */ + process_while_blocked = 0; /* Don't spawn any thread if the user selected a single thread: * we'll handle I/O directly from the main thread. */ @@ -2970,6 +2974,7 @@ int handleClientsWithPendingWritesUsingThreads(void) { int postponeClientRead(client *c) { if (io_threads_active && server.io_threads_do_reads && + !process_while_blocked && !(c->flags & (CLIENT_MASTER|CLIENT_SLAVE|CLIENT_PENDING_READ))) { c->flags |= CLIENT_PENDING_READ; From 7c07841632500dfdff31f5a69e5bd140abea646c Mon Sep 17 00:00:00 2001 From: "zhaozhao.zz" Date: Sun, 15 Mar 2020 23:30:25 +0800 Subject: [PATCH 153/225] Threaded IO: bugfix client kill may crash redis --- src/networking.c | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/src/networking.c b/src/networking.c index c7dd2f23c..0690bbdf6 100644 --- a/src/networking.c +++ b/src/networking.c @@ -3040,16 +3040,22 @@ int handleClientsWithPendingReadsUsingThreads(void) { if (tio_debug) printf("I/O READ All threads finshed\n"); /* Run the list of clients again to process the new buffers. */ - listRewind(server.clients_pending_read,&li); - while((ln = listNext(&li))) { + while(listLength(server.clients_pending_read)) { + ln = listFirst(server.clients_pending_read); client *c = listNodeValue(ln); c->flags &= ~CLIENT_PENDING_READ; + listDelNode(server.clients_pending_read,ln); + if (c->flags & CLIENT_PENDING_COMMAND) { - c->flags &= ~ CLIENT_PENDING_COMMAND; - processCommandAndResetClient(c); + c->flags &= ~CLIENT_PENDING_COMMAND; + if (processCommandAndResetClient(c) == C_ERR) { + /* If the client is no longer valid, we avoid + * processing the client later. So we just go + * to the next. */ + continue; + } } processInputBufferAndReplicate(c); } - listEmpty(server.clients_pending_read); return processed; } From 95324b8190e5338c287dff32ce65d50a29c981eb Mon Sep 17 00:00:00 2001 From: artix Date: Tue, 18 Feb 2020 10:46:10 +0100 Subject: [PATCH 154/225] Support Redis Cluster Proxy PROXY INFO command --- src/redis-cli.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/redis-cli.c b/src/redis-cli.c index 7ad80c0a1..7e440d67c 100644 --- a/src/redis-cli.c +++ b/src/redis-cli.c @@ -1292,7 +1292,11 @@ static int cliSendCommand(int argc, char **argv, long repeat) { (argc == 3 && !strcasecmp(command,"latency") && !strcasecmp(argv[1],"graph")) || (argc == 2 && !strcasecmp(command,"latency") && - !strcasecmp(argv[1],"doctor"))) + !strcasecmp(argv[1],"doctor")) || + /* Format PROXY INFO command for Redis Cluster Proxy: + * https://github.com/artix75/redis-cluster-proxy */ + (argc >= 2 && !strcasecmp(command,"proxy") && + !strcasecmp(argv[1],"info"))) { output_raw = 1; } From 2dd1ca6af0d0d6514e8c18eeba86b5f1b7a55979 Mon Sep 17 00:00:00 2001 From: hwware Date: Fri, 20 Mar 2020 02:40:54 -0400 Subject: [PATCH 155/225] add missing commands in cluster help --- src/cluster.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/cluster.c b/src/cluster.c index 5f63d2b8f..72755823a 100644 --- a/src/cluster.c +++ b/src/cluster.c @@ -4261,7 +4261,7 @@ void clusterCommand(client *c) { "FORGET -- Remove a node from the cluster.", "GETKEYSINSLOT -- Return key names stored by current node in a slot.", "FLUSHSLOTS -- Delete current node own slots information.", -"INFO - Return onformation about the cluster.", +"INFO - Return information about the cluster.", "KEYSLOT -- Return the hash slot for .", "MEET [bus-port] -- Connect nodes into a working cluster.", "MYID -- Return the node id.", @@ -4272,6 +4272,7 @@ void clusterCommand(client *c) { "SET-config-epoch - Set config epoch of current node.", "SETSLOT (importing|migrating|stable|node ) -- Set slot state.", "REPLICAS -- Return replicas.", +"SAVECONFIG - Force saving cluster configuration on disk.", "SLOTS -- Return information about slots range mappings. Each range is made of:", " start, end, master and replicas IP addresses, ports and ids", NULL From c7524a7e447d92775c1fb61d8f0640955bce952b Mon Sep 17 00:00:00 2001 From: hwware Date: Mon, 23 Mar 2020 01:04:49 -0400 Subject: [PATCH 156/225] clean CLIENT_TRACKING_CACHING flag when disabled caching --- src/tracking.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tracking.c b/src/tracking.c index 45f83103a..5c1f48cba 100644 --- a/src/tracking.c +++ b/src/tracking.c @@ -94,7 +94,7 @@ void disableTracking(client *c) { server.tracking_clients--; c->flags &= ~(CLIENT_TRACKING|CLIENT_TRACKING_BROKEN_REDIR| CLIENT_TRACKING_BCAST|CLIENT_TRACKING_OPTIN| - CLIENT_TRACKING_OPTOUT); + CLIENT_TRACKING_OPTOUT|CLIENT_TRACKING_CACHING); } } From 81e8686cc7e1ed9f2c5d306b93cfa7462a05c7d2 Mon Sep 17 00:00:00 2001 From: hwware Date: Mon, 23 Mar 2020 01:07:46 -0400 Subject: [PATCH 157/225] remove redundant Semicolon --- src/tracking.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tracking.c b/src/tracking.c index 5c1f48cba..6f7929430 100644 --- a/src/tracking.c +++ b/src/tracking.c @@ -271,7 +271,7 @@ void trackingInvalidateKey(robj *keyobj) { trackingRememberKeyToBroadcast(sdskey,sdslen(sdskey)); rax *ids = raxFind(TrackingTable,(unsigned char*)sdskey,sdslen(sdskey)); - if (ids == raxNotFound) return;; + if (ids == raxNotFound) return; raxIterator ri; raxStart(&ri,ids); From 87dbd8f54cdf5f0fb97851cf51dffe1bb9a16a1c Mon Sep 17 00:00:00 2001 From: Yossi Gottlieb Date: Sun, 22 Mar 2020 14:42:03 +0200 Subject: [PATCH 158/225] Conns: Fix connClose() / connAccept() behavior. We assume accept handlers may choose to reject a connection and close it, but connAccept() callers can't distinguish between this state and other error states requiring connClose(). This makes it safe (and mandatory!) to always call connClose() if connAccept() fails, and safe for accept handlers to close connections (which will defer). --- src/connection.c | 12 ++++++++--- src/connection.h | 15 ++++++++++---- src/connhelpers.h | 53 +++++++++++++++++++++++++---------------------- 3 files changed, 48 insertions(+), 32 deletions(-) diff --git a/src/connection.c b/src/connection.c index 58d86c31b..2015c9195 100644 --- a/src/connection.c +++ b/src/connection.c @@ -152,7 +152,7 @@ static void connSocketClose(connection *conn) { /* If called from within a handler, schedule the close but * keep the connection until the handler returns. */ - if (conn->flags & CONN_FLAG_IN_HANDLER) { + if (connHasRefs(conn)) { conn->flags |= CONN_FLAG_CLOSE_SCHEDULED; return; } @@ -183,10 +183,16 @@ static int connSocketRead(connection *conn, void *buf, size_t buf_len) { } static int connSocketAccept(connection *conn, ConnectionCallbackFunc accept_handler) { + int ret = C_OK; + if (conn->state != CONN_STATE_ACCEPTING) return C_ERR; conn->state = CONN_STATE_CONNECTED; - if (!callHandler(conn, accept_handler)) return C_ERR; - return C_OK; + + connIncrRefs(conn); + if (!callHandler(conn, accept_handler)) ret = C_ERR; + connDecrRefs(conn); + + return ret; } /* Register a write handler, to be called when the connection is writable. diff --git a/src/connection.h b/src/connection.h index 97622f8d6..db09dfd83 100644 --- a/src/connection.h +++ b/src/connection.h @@ -45,9 +45,8 @@ typedef enum { CONN_STATE_ERROR } ConnectionState; -#define CONN_FLAG_IN_HANDLER (1<<0) /* A handler execution is in progress */ -#define CONN_FLAG_CLOSE_SCHEDULED (1<<1) /* Closed scheduled by a handler */ -#define CONN_FLAG_WRITE_BARRIER (1<<2) /* Write barrier requested */ +#define CONN_FLAG_CLOSE_SCHEDULED (1<<0) /* Closed scheduled by a handler */ +#define CONN_FLAG_WRITE_BARRIER (1<<1) /* Write barrier requested */ typedef void (*ConnectionCallbackFunc)(struct connection *conn); @@ -70,7 +69,8 @@ typedef struct ConnectionType { struct connection { ConnectionType *type; ConnectionState state; - int flags; + short int flags; + short int refs; int last_errno; void *private_data; ConnectionCallbackFunc conn_handler; @@ -88,6 +88,13 @@ struct connection { * connAccept() may directly call accept_handler(), or return and call it * at a later time. This behavior is a bit awkward but aims to reduce the need * to wait for the next event loop, if no additional handshake is required. + * + * IMPORTANT: accept_handler may decide to close the connection, calling connClose(). + * To make this safe, the connection is only marked with CONN_FLAG_CLOSE_SCHEDULED + * in this case, and connAccept() returns with an error. + * + * connAccept() callers must always check the return value and on error (C_ERR) + * a connClose() must be called. */ static inline int connAccept(connection *conn, ConnectionCallbackFunc accept_handler) { diff --git a/src/connhelpers.h b/src/connhelpers.h index f237c9b1d..86250d09e 100644 --- a/src/connhelpers.h +++ b/src/connhelpers.h @@ -37,46 +37,49 @@ * implementations (currently sockets in connection.c and TLS in tls.c). * * Currently helpers implement the mechanisms for invoking connection - * handlers, tracking in-handler states and dealing with deferred - * destruction (if invoked by a handler). + * handlers and tracking connection references, to allow safe destruction + * of connections from within a handler. */ -/* Called whenever a handler is invoked on a connection and sets the - * CONN_FLAG_IN_HANDLER flag to indicate we're in a handler context. +/* Incremenet connection references. * - * An attempt to close a connection while CONN_FLAG_IN_HANDLER is - * set will result with deferred close, i.e. setting the CONN_FLAG_CLOSE_SCHEDULED - * instead of destructing it. + * Inside a connection handler, we guarantee refs >= 1 so it is always + * safe to connClose(). + * + * In other cases where we don't want to prematurely lose the connection, + * it can go beyond 1 as well; currently it is only done by connAccept(). */ -static inline void enterHandler(connection *conn) { - conn->flags |= CONN_FLAG_IN_HANDLER; +static inline void connIncrRefs(connection *conn) { + conn->refs++; } -/* Called whenever a handler returns. This unsets the CONN_FLAG_IN_HANDLER - * flag and performs actual close/destruction if a deferred close was - * scheduled by the handler. +/* Decrement connection references. + * + * Note that this is not intended to provide any automatic free logic! + * callHandler() takes care of that for the common flows, and anywhere an + * explicit connIncrRefs() is used, the caller is expected to take care of + * that. */ -static inline int exitHandler(connection *conn) { - conn->flags &= ~CONN_FLAG_IN_HANDLER; - if (conn->flags & CONN_FLAG_CLOSE_SCHEDULED) { - connClose(conn); - return 0; - } - return 1; + +static inline void connDecrRefs(connection *conn) { + conn->refs--; +} + +static inline int connHasRefs(connection *conn) { + return conn->refs; } /* Helper for connection implementations to call handlers: - * 1. Mark the handler in use. + * 1. Increment refs to protect the connection. * 2. Execute the handler (if set). - * 3. Mark the handler as NOT in use and perform deferred close if was - * requested by the handler at any time. + * 3. Decrement refs and perform deferred close, if refs==0. */ static inline int callHandler(connection *conn, ConnectionCallbackFunc handler) { - conn->flags |= CONN_FLAG_IN_HANDLER; + connIncrRefs(conn); if (handler) handler(conn); - conn->flags &= ~CONN_FLAG_IN_HANDLER; + connDecrRefs(conn); if (conn->flags & CONN_FLAG_CLOSE_SCHEDULED) { - connClose(conn); + if (!connHasRefs(conn)) connClose(conn); return 0; } return 1; From 50dcd9f96d133b740aa04a3877d67fb890630bc8 Mon Sep 17 00:00:00 2001 From: Yossi Gottlieb Date: Sun, 22 Mar 2020 14:46:16 +0200 Subject: [PATCH 159/225] Cluster: fix misleading accept errors. --- src/cluster.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/cluster.c b/src/cluster.c index 72755823a..a2e9ff5b6 100644 --- a/src/cluster.c +++ b/src/cluster.c @@ -681,9 +681,10 @@ void clusterAcceptHandler(aeEventLoop *el, int fd, void *privdata, int mask) { * or schedule it for later depending on connection implementation. */ if (connAccept(conn, clusterConnAcceptHandler) == C_ERR) { - serverLog(LL_VERBOSE, - "Error accepting cluster node connection: %s", - connGetLastError(conn)); + if (connGetState(conn) == CONN_STATE_ERROR) + serverLog(LL_VERBOSE, + "Error accepting cluster node connection: %s", + connGetLastError(conn)); connClose(conn); return; } From cdcab0e820710c17eb5cdaf0fa532908e3417df1 Mon Sep 17 00:00:00 2001 From: Yossi Gottlieb Date: Sun, 22 Mar 2020 14:47:44 +0200 Subject: [PATCH 160/225] Fix crashes related to failed/rejected accepts. --- src/networking.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/networking.c b/src/networking.c index 69d59a59b..a550e4040 100644 --- a/src/networking.c +++ b/src/networking.c @@ -786,7 +786,7 @@ void clientAcceptHandler(connection *conn) { serverLog(LL_WARNING, "Error accepting a client connection: %s", connGetLastError(conn)); - freeClient(c); + freeClientAsync(c); return; } @@ -828,7 +828,7 @@ void clientAcceptHandler(connection *conn) { /* Nothing to do, Just to avoid the warning... */ } server.stat_rejected_conn++; - freeClient(c); + freeClientAsync(c); return; } } @@ -887,9 +887,10 @@ static void acceptCommonHandler(connection *conn, int flags, char *ip) { */ if (connAccept(conn, clientAcceptHandler) == C_ERR) { char conninfo[100]; - serverLog(LL_WARNING, - "Error accepting a client connection: %s (conn: %s)", - connGetLastError(conn), connGetInfo(conn, conninfo, sizeof(conninfo))); + if (connGetState(conn) == CONN_STATE_ERROR) + serverLog(LL_WARNING, + "Error accepting a client connection: %s (conn: %s)", + connGetLastError(conn), connGetInfo(conn, conninfo, sizeof(conninfo))); freeClient(connGetPrivateData(conn)); return; } From eb80887936cd57223bda4776129bab157b09c5a7 Mon Sep 17 00:00:00 2001 From: hwware Date: Wed, 18 Mar 2020 09:33:52 -0400 Subject: [PATCH 161/225] fix potentical memory leak in redis-cli --- src/redis-cli.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/redis-cli.c b/src/redis-cli.c index 7e440d67c..72480d08c 100644 --- a/src/redis-cli.c +++ b/src/redis-cli.c @@ -1993,6 +1993,8 @@ static void repl(void) { if (config.eval) { config.eval_ldb = 1; config.output = OUTPUT_RAW; + sdsfreesplitres(argv,argc); + linenoiseFree(line); return; /* Return to evalMode to restart the session. */ } else { printf("Use 'restart' only in Lua debugging mode."); From f2f3dc5e73f93ec8c5b9efe360adbf06da0ea0bd Mon Sep 17 00:00:00 2001 From: Guy Benoish Date: Wed, 18 Mar 2020 18:34:27 +0530 Subject: [PATCH 162/225] Allow RM_GetContextFlags to work with ctx==NULL --- src/module.c | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/src/module.c b/src/module.c index 74da6c24d..d6b055f78 100644 --- a/src/module.c +++ b/src/module.c @@ -1848,20 +1848,22 @@ int RM_GetContextFlags(RedisModuleCtx *ctx) { int flags = 0; /* Client specific flags */ - if (ctx->client) { - if (ctx->client->flags & CLIENT_LUA) - flags |= REDISMODULE_CTX_FLAGS_LUA; - if (ctx->client->flags & CLIENT_MULTI) - flags |= REDISMODULE_CTX_FLAGS_MULTI; - /* Module command recieved from MASTER, is replicated. */ - if (ctx->client->flags & CLIENT_MASTER) - flags |= REDISMODULE_CTX_FLAGS_REPLICATED; - } + if (ctx) { + if (ctx->client) { + if (ctx->client->flags & CLIENT_LUA) + flags |= REDISMODULE_CTX_FLAGS_LUA; + if (ctx->client->flags & CLIENT_MULTI) + flags |= REDISMODULE_CTX_FLAGS_MULTI; + /* Module command recieved from MASTER, is replicated. */ + if (ctx->client->flags & CLIENT_MASTER) + flags |= REDISMODULE_CTX_FLAGS_REPLICATED; + } - /* For DIRTY flags, we need the blocked client if used */ - client *c = ctx->blocked_client ? ctx->blocked_client->client : ctx->client; - if (c && (c->flags & (CLIENT_DIRTY_CAS|CLIENT_DIRTY_EXEC))) { - flags |= REDISMODULE_CTX_FLAGS_MULTI_DIRTY; + /* For DIRTY flags, we need the blocked client if used */ + client *c = ctx->blocked_client ? ctx->blocked_client->client : ctx->client; + if (c && (c->flags & (CLIENT_DIRTY_CAS|CLIENT_DIRTY_EXEC))) { + flags |= REDISMODULE_CTX_FLAGS_MULTI_DIRTY; + } } if (server.cluster_enabled) From 50f8f9504b2ba24edc646cfb32dac2a496ba34cd Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 23 Mar 2020 11:17:50 +0100 Subject: [PATCH 163/225] Modules: updated function doc after #7003. --- src/module.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/module.c b/src/module.c index d6b055f78..6f61a5ca8 100644 --- a/src/module.c +++ b/src/module.c @@ -1795,7 +1795,12 @@ int RM_GetSelectedDb(RedisModuleCtx *ctx) { * current request context (whether the client is a Lua script or in a MULTI), * and about the Redis instance in general, i.e replication and persistence. * - * The available flags are: + * It is possible to call this function even with a NULL context, however + * in this case the following flags will not be reported: + * + * * LUA, MULTI, REPLICATED, DIRTY (see below for more info). + * + * Available flags and their meaning: * * * REDISMODULE_CTX_FLAGS_LUA: The command is running in a Lua script * From b3e4abf06ea01506a809898552845a191fb52cbf Mon Sep 17 00:00:00 2001 From: "bodong.ybd" Date: Wed, 4 Mar 2020 20:51:45 +0800 Subject: [PATCH 164/225] Added BITFIELD_RO variants for read-only operations. --- src/bitops.c | 19 ++++++++++++++++++- src/server.c | 4 ++++ src/server.h | 1 + tests/unit/bitfield.tcl | 31 +++++++++++++++++++++++++++++++ 4 files changed, 54 insertions(+), 1 deletion(-) diff --git a/src/bitops.c b/src/bitops.c index ee1ce0460..ffb330013 100644 --- a/src/bitops.c +++ b/src/bitops.c @@ -902,6 +902,9 @@ void bitposCommand(client *c) { * OVERFLOW [WRAP|SAT|FAIL] */ +#define BITFIELD_COMMON (1<<0) +#define BITFIELD_READONLY (1<<1) + struct bitfieldOp { uint64_t offset; /* Bitfield offset. */ int64_t i64; /* Increment amount (INCRBY) or SET value */ @@ -911,7 +914,7 @@ struct bitfieldOp { int sign; /* True if signed, otherwise unsigned op. */ }; -void bitfieldCommand(client *c) { +void bitfieldGeneric(client *c, int flags) { robj *o; size_t bitoffset; int j, numops = 0, changes = 0; @@ -999,6 +1002,12 @@ void bitfieldCommand(client *c) { return; } } else { + if (flags & BITFIELD_READONLY) { + zfree(ops); + addReplyError(c, "bitfield_ro only support get subcommand"); + return; + } + /* Lookup by making room up to the farest bit reached by * this operation. */ if ((o = lookupStringForBitCommand(c, @@ -1129,3 +1138,11 @@ void bitfieldCommand(client *c) { } zfree(ops); } + +void bitfieldCommand(client *c) { + bitfieldGeneric(c, BITFIELD_COMMON); +} + +void bitfieldroCommand(client *c) { + bitfieldGeneric(c, BITFIELD_READONLY); +} diff --git a/src/server.c b/src/server.c index 4b010b870..f5fb339f9 100644 --- a/src/server.c +++ b/src/server.c @@ -238,6 +238,10 @@ struct redisCommand redisCommandTable[] = { "write use-memory @bitmap", 0,NULL,1,1,1,0,0,0}, + {"bitfield_ro",bitfieldroCommand,-2, + "read-only fast @bitmap", + 0,NULL,1,1,1,0,0,0}, + {"setrange",setrangeCommand,4, "write use-memory @string", 0,NULL,1,1,1,0,0,0}, diff --git a/src/server.h b/src/server.h index fdfe5b8ea..ed4707d66 100644 --- a/src/server.h +++ b/src/server.h @@ -2177,6 +2177,7 @@ void existsCommand(client *c); void setbitCommand(client *c); void getbitCommand(client *c); void bitfieldCommand(client *c); +void bitfieldroCommand(client *c); void setrangeCommand(client *c); void getrangeCommand(client *c); void incrCommand(client *c); diff --git a/tests/unit/bitfield.tcl b/tests/unit/bitfield.tcl index d76452b1b..819d8f36d 100644 --- a/tests/unit/bitfield.tcl +++ b/tests/unit/bitfield.tcl @@ -199,3 +199,34 @@ start_server {tags {"bitops"}} { r del mystring } } + +start_server {tags {"repl"}} { + start_server {} { + set master [srv -1 client] + set master_host [srv -1 host] + set master_port [srv -1 port] + set slave [srv 0 client] + + test {setup slave} { + $slave slaveof $master_host $master_port + wait_for_condition 50 100 { + [s 0 master_link_status] eq {up} + } else { + fail "Replication not started." + } + } + + test {write on master, read on slave} { + $master del bits + assert_equal 0 [$master bitfield bits set u8 0 255] + assert_equal 255 [$master bitfield bits set u8 0 100] + wait_for_ofs_sync $master $slave + assert_equal 100 [$slave bitfield_ro bits get u8 0] + } + + test {bitfield_ro with write option} { + catch {$slave bitfield_ro bits set u8 0 100 get u8 0} err + assert_match {*ERR bitfield_ro only support get subcommand*} $err + } + } +} From ec9cf002d537d2d565b3d91c116d2bab0a9a5fd4 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 23 Mar 2020 11:28:09 +0100 Subject: [PATCH 165/225] Minor changes to BITFIELD_RO PR #6951. --- src/bitops.c | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/bitops.c b/src/bitops.c index ffb330013..d4e82c937 100644 --- a/src/bitops.c +++ b/src/bitops.c @@ -902,8 +902,8 @@ void bitposCommand(client *c) { * OVERFLOW [WRAP|SAT|FAIL] */ -#define BITFIELD_COMMON (1<<0) -#define BITFIELD_READONLY (1<<1) +#define BITFIELD_FLAG_NONE 0 +#define BITFIELD_FLAG_READONLY (1<<0) struct bitfieldOp { uint64_t offset; /* Bitfield offset. */ @@ -914,6 +914,9 @@ struct bitfieldOp { int sign; /* True if signed, otherwise unsigned op. */ }; +/* This implements both the BITFIELD command and the BITFIELD_RO command + * when flags is set to BITFIELD_FLAG_READONLY: in this case only the + * GET subcommand is allowed, other subcommands will return an error. */ void bitfieldGeneric(client *c, int flags) { robj *o; size_t bitoffset; @@ -1002,9 +1005,9 @@ void bitfieldGeneric(client *c, int flags) { return; } } else { - if (flags & BITFIELD_READONLY) { + if (flags & BITFIELD_FLAG_READONLY) { zfree(ops); - addReplyError(c, "bitfield_ro only support get subcommand"); + addReplyError(c, "BITFIELD_RO only support the GET subcommand"); return; } @@ -1140,9 +1143,9 @@ void bitfieldGeneric(client *c, int flags) { } void bitfieldCommand(client *c) { - bitfieldGeneric(c, BITFIELD_COMMON); + bitfieldGeneric(c, BITFIELD_FLAG_NONE); } void bitfieldroCommand(client *c) { - bitfieldGeneric(c, BITFIELD_READONLY); + bitfieldGeneric(c, BITFIELD_FLAG_READONLY); } From 8783304a2da57561e423da280cc08c9a8e4c44ad Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 23 Mar 2020 11:47:37 +0100 Subject: [PATCH 166/225] Abort transactions after -READONLY error. Fix #7014. --- src/server.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/server.c b/src/server.c index f5fb339f9..ddc90b3dd 100644 --- a/src/server.c +++ b/src/server.c @@ -3509,6 +3509,7 @@ int processCommand(client *c) { !(c->flags & CLIENT_MASTER) && c->cmd->flags & CMD_WRITE) { + flagTransaction(c); addReply(c, shared.roslaveerr); return C_OK; } From 70a98a43ea5e4544f83bdea5257a3733c8afe188 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 23 Mar 2020 12:00:46 +0100 Subject: [PATCH 167/225] Fix BITFIELD_RO test. --- src/bitops.c | 2 +- tests/unit/bitfield.tcl | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/bitops.c b/src/bitops.c index d4e82c937..f78e4fd34 100644 --- a/src/bitops.c +++ b/src/bitops.c @@ -1007,7 +1007,7 @@ void bitfieldGeneric(client *c, int flags) { } else { if (flags & BITFIELD_FLAG_READONLY) { zfree(ops); - addReplyError(c, "BITFIELD_RO only support the GET subcommand"); + addReplyError(c, "BITFIELD_RO only supports the GET subcommand"); return; } diff --git a/tests/unit/bitfield.tcl b/tests/unit/bitfield.tcl index 819d8f36d..1f2f6395e 100644 --- a/tests/unit/bitfield.tcl +++ b/tests/unit/bitfield.tcl @@ -207,7 +207,7 @@ start_server {tags {"repl"}} { set master_port [srv -1 port] set slave [srv 0 client] - test {setup slave} { + test {BITFIELD: setup slave} { $slave slaveof $master_host $master_port wait_for_condition 50 100 { [s 0 master_link_status] eq {up} @@ -216,7 +216,7 @@ start_server {tags {"repl"}} { } } - test {write on master, read on slave} { + test {BITFIELD: write on master, read on slave} { $master del bits assert_equal 0 [$master bitfield bits set u8 0 255] assert_equal 255 [$master bitfield bits set u8 0 100] @@ -224,9 +224,9 @@ start_server {tags {"repl"}} { assert_equal 100 [$slave bitfield_ro bits get u8 0] } - test {bitfield_ro with write option} { + test {BITFIELD_RO fails when write option is used} { catch {$slave bitfield_ro bits set u8 0 100 get u8 0} err - assert_match {*ERR bitfield_ro only support get subcommand*} $err + assert_match {*ERR BITFIELD_RO only supports the GET subcommand*} $err } } } From 34b89832200fa8bce5d739cb732d1bddfc25a83e Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 23 Mar 2020 16:17:35 +0100 Subject: [PATCH 168/225] Improve comments of replicationCacheMasterUsingMyself(). --- src/replication.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/replication.c b/src/replication.c index 31e14d7fe..49c38f73f 100644 --- a/src/replication.c +++ b/src/replication.c @@ -2725,9 +2725,14 @@ void replicationCacheMaster(client *c) { * current offset if no data was lost during the failover. So we use our * current replication ID and offset in order to synthesize a cached master. */ void replicationCacheMasterUsingMyself(void) { + /* This will be used to populate the field server.master->reploff + * by replicationCreateMasterClient(). We'll later set the created + * master as server.cached_master, so the replica will use such + * offset for PSYNC. */ + server.master_initial_offset = server.master_repl_offset; + /* The master client we create can be set to any DBID, because * the new master will start its replication stream with SELECT. */ - server.master_initial_offset = server.master_repl_offset; replicationCreateMasterClient(NULL,-1); /* Use our own ID / offset. */ From e43cd8316f16525371f06e29512e66daa24c8f74 Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Mon, 23 Mar 2020 20:13:52 +0200 Subject: [PATCH 169/225] MULTI/EXEC during LUA script timeout are messed up Redis refusing to run MULTI or EXEC during script timeout may cause partial transactions to run. 1) if the client sends MULTI+commands+EXEC in pipeline without waiting for response, but these arrive to the shards partially while there's a busy script, and partially after it eventually finishes: we'll end up running only part of the transaction (since multi was ignored, and exec would fail). 2) similar to the above if EXEC arrives during busy script, it'll be ignored and the client state remains in a transaction. the 3rd test which i added for a case where MULTI and EXEC are ok, and only the body arrives during busy script was already handled correctly since processCommand calls flagTransaction --- src/server.c | 1 + tests/unit/multi.tcl | 72 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 73 insertions(+) diff --git a/src/server.c b/src/server.c index ddc90b3dd..5a54a3abb 100644 --- a/src/server.c +++ b/src/server.c @@ -3553,6 +3553,7 @@ int processCommand(client *c) { c->cmd->proc != authCommand && c->cmd->proc != helloCommand && c->cmd->proc != replconfCommand && + c->cmd->proc != multiCommand && c->cmd->proc != execCommand && c->cmd->proc != discardCommand && !(c->cmd->proc == shutdownCommand && c->argc == 2 && tolower(((char*)c->argv[1]->ptr)[0]) == 'n') && diff --git a/tests/unit/multi.tcl b/tests/unit/multi.tcl index 9fcef71d6..55f18bec8 100644 --- a/tests/unit/multi.tcl +++ b/tests/unit/multi.tcl @@ -320,4 +320,76 @@ start_server {tags {"multi"}} { $rd close r ping } {PONG} + + test {MULTI and script timeout} { + # check that if MULTI arrives during timeout, it is either refused, or + # allowed to pass, and we don't end up executing half of the transaction + set rd1 [redis_deferring_client] + set rd2 [redis_deferring_client] + r config set lua-time-limit 10 + r set xx 1 + $rd1 eval {while true do end} 0 + after 200 + catch { $rd2 multi; $rd2 read } e + catch { $rd2 incr xx; $rd2 read } e + r script kill + after 200 ; # Give some time to Lua to call the hook again... + catch { $rd2 incr xx; $rd2 read } e + catch { $rd2 exec; $rd2 read } e + set xx [r get xx] + # make sure that either the whole transcation passed or none of it (we actually expect none) + assert { $xx == 1 || $xx == 3} + # check that the connection is no longer in multi state + $rd2 ping asdf + set pong [$rd2 read] + assert_equal $pong "asdf" + } + + test {EXEC and script timeout} { + # check that if EXEC arrives during timeout, we don't end up executing + # half of the transaction, and also that we exit the multi state + set rd1 [redis_deferring_client] + set rd2 [redis_deferring_client] + r config set lua-time-limit 10 + r set xx 1 + catch { $rd2 multi; $rd2 read } e + catch { $rd2 incr xx; $rd2 read } e + $rd1 eval {while true do end} 0 + after 200 + catch { $rd2 incr xx; $rd2 read } e + catch { $rd2 exec; $rd2 read } e + r script kill + after 200 ; # Give some time to Lua to call the hook again... + set xx [r get xx] + # make sure that either the whole transcation passed or none of it (we actually expect none) + assert { $xx == 1 || $xx == 3} + # check that the connection is no longer in multi state + $rd2 ping asdf + set pong [$rd2 read] + assert_equal $pong "asdf" + } + + test {MULTI-EXEC body and script timeout} { + # check that we don't run an imcomplete transaction due to some commands + # arriving during busy script + set rd1 [redis_deferring_client] + set rd2 [redis_deferring_client] + r config set lua-time-limit 10 + r set xx 1 + catch { $rd2 multi; $rd2 read } e + catch { $rd2 incr xx; $rd2 read } e + $rd1 eval {while true do end} 0 + after 200 + catch { $rd2 incr xx; $rd2 read } e + r script kill + after 200 ; # Give some time to Lua to call the hook again... + catch { $rd2 exec; $rd2 read } e + set xx [r get xx] + # make sure that either the whole transcation passed or none of it (we actually expect none) + assert { $xx == 1 || $xx == 3} + # check that the connection is no longer in multi state + $rd2 ping asdf + set pong [$rd2 read] + assert_equal $pong "asdf" + } } From 8caa2714768947082fe876891d93431a3f69efdf Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 25 Mar 2020 12:46:59 +0100 Subject: [PATCH 170/225] Explain why we allow transactions in -BUSY state. Related to #7022. --- src/server.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/server.c b/src/server.c index 5a54a3abb..65a01db57 100644 --- a/src/server.c +++ b/src/server.c @@ -3548,12 +3548,19 @@ int processCommand(client *c) { return C_OK; } - /* Lua script too slow? Only allow a limited number of commands. */ + /* Lua script too slow? Only allow a limited number of commands. + * Note that we need to allow the transactions commands, otherwise clients + * sending a transaction with pipelining without error checking, may have + * the MULTI plus a few initial commands refused, then the timeout + * condition resolves, and the bottom-half of the transaction gets + * executed, see Github PR #7022. */ if (server.lua_timedout && c->cmd->proc != authCommand && c->cmd->proc != helloCommand && c->cmd->proc != replconfCommand && - c->cmd->proc != multiCommand && c->cmd->proc != execCommand && c->cmd->proc != discardCommand && + c->cmd->proc != multiCommand && + c->cmd->proc != execCommand && + c->cmd->proc != discardCommand && !(c->cmd->proc == shutdownCommand && c->argc == 2 && tolower(((char*)c->argv[1]->ptr)[0]) == 'n') && From 5f72f696884c6007704e4439f850aa6ce3662013 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 24 Mar 2020 11:02:40 +0100 Subject: [PATCH 171/225] PSYNC2: meaningful offset implemented. A very commonly signaled operational problem with Redis master-replicas sets is that, once the master becomes unavailable for some reason, especially because of network problems, many times it wont be able to perform a partial resynchronization with the new master, once it rejoins the partition, for the following reason: 1. The master becomes isolated, however it keeps sending PINGs to the replicas. Such PINGs will never be received since the link connection is actually already severed. 2. On the other side, one of the replicas will turn into the new master, setting its secondary replication ID offset to the one of the last command received from the old master: this offset will not include the PINGs sent by the master once the link was already disconnected. 3. When the master rejoins the partion and is turned into a replica, its offset will be too advanced because of the PINGs, so a PSYNC will fail, and a full synchronization will be required. Related to issue #7002 and other discussion we had in the past around this problem. --- src/replication.c | 36 +++++++++++++++++++++++++++++++++++- src/server.c | 4 ++++ src/server.h | 1 + 3 files changed, 40 insertions(+), 1 deletion(-) diff --git a/src/replication.c b/src/replication.c index 49c38f73f..e62afb2fb 100644 --- a/src/replication.c +++ b/src/replication.c @@ -162,6 +162,7 @@ void feedReplicationBacklog(void *ptr, size_t len) { unsigned char *p = ptr; server.master_repl_offset += len; + server.master_repl_meaningful_offset = server.master_repl_offset; /* This is a circular buffer, so write as much data we can at every * iteration and rewind the "idx" index if we reach the limit. */ @@ -1768,6 +1769,7 @@ void readSyncBulkPayload(connection *conn) { * we are starting a new history. */ memcpy(server.replid,server.master->replid,sizeof(server.replid)); server.master_repl_offset = server.master->reploff; + server.master_repl_meaningful_offset = server.master->reploff; clearReplicationId2(); /* Let's create the replication backlog if needed. Slaves need to @@ -2725,12 +2727,37 @@ void replicationCacheMaster(client *c) { * current offset if no data was lost during the failover. So we use our * current replication ID and offset in order to synthesize a cached master. */ void replicationCacheMasterUsingMyself(void) { + serverLog(LL_NOTICE, + "Before turning into a replica, using my own master parameters " + "to synthesize a cached master: I may be able to synchronize with " + "the new master with just a partial transfer."); + /* This will be used to populate the field server.master->reploff * by replicationCreateMasterClient(). We'll later set the created * master as server.cached_master, so the replica will use such * offset for PSYNC. */ server.master_initial_offset = server.master_repl_offset; + /* However if the "meaningful" offset, that is the offset without + * the final PINGs in the stream, is different, use this instead: + * often when the master is no longer reachable, replicas will never + * receive the PINGs, however the master will end with an incremented + * offset because of the PINGs and will not be able to incrementally + * PSYNC with the new master. */ + if (server.master_repl_offset > server.master_repl_meaningful_offset) { + long long delta = server.master_repl_offset - + server.master_repl_meaningful_offset; + serverLog(LL_NOTICE, + "Using the meaningful offset %lld instead of %lld to exclude " + "the final PINGs (%lld bytes difference)", + server.master_repl_meaningful_offset, + server.master_repl_offset, + delta); + server.master_initial_offset = server.master_repl_meaningful_offset; + server.repl_backlog_histlen -= delta; + if (server.repl_backlog_histlen < 0) server.repl_backlog_histlen = 0; + } + /* The master client we create can be set to any DBID, because * the new master will start its replication stream with SELECT. */ replicationCreateMasterClient(NULL,-1); @@ -2742,7 +2769,6 @@ void replicationCacheMasterUsingMyself(void) { unlinkClient(server.master); server.cached_master = server.master; server.master = NULL; - serverLog(LL_NOTICE,"Before turning into a replica, using my master parameters to synthesize a cached master: I may be able to synchronize with the new master with just a partial transfer."); } /* Free a cached master, called when there are no longer the conditions for @@ -3122,10 +3148,18 @@ void replicationCron(void) { clientsArePaused(); if (!manual_failover_in_progress) { + long long before_ping = server.master_repl_meaningful_offset; ping_argv[0] = createStringObject("PING",4); replicationFeedSlaves(server.slaves, server.slaveseldb, ping_argv, 1); decrRefCount(ping_argv[0]); + /* The server.master_repl_meaningful_offset variable represents + * the offset of the replication stream without the pending PINGs. + * This is useful to set the right replication offset for PSYNC + * when the master is turned into a replica. Otherwise pending + * PINGs may not allow it to perform an incremental sync with the + * new master. */ + server.master_repl_meaningful_offset = before_ping; } } diff --git a/src/server.c b/src/server.c index 65a01db57..84439461e 100644 --- a/src/server.c +++ b/src/server.c @@ -2355,6 +2355,7 @@ void initServerConfig(void) { server.repl_syncio_timeout = CONFIG_REPL_SYNCIO_TIMEOUT; server.repl_down_since = 0; /* Never connected, repl is down since EVER. */ server.master_repl_offset = 0; + server.master_repl_meaningful_offset = 0; /* Replication partial resync backlog */ server.repl_backlog = NULL; @@ -4398,6 +4399,7 @@ sds genRedisInfoString(const char *section) { "master_replid:%s\r\n" "master_replid2:%s\r\n" "master_repl_offset:%lld\r\n" + "master_repl_meaningful_offset:%lld\r\n" "second_repl_offset:%lld\r\n" "repl_backlog_active:%d\r\n" "repl_backlog_size:%lld\r\n" @@ -4406,6 +4408,7 @@ sds genRedisInfoString(const char *section) { server.replid, server.replid2, server.master_repl_offset, + server.master_repl_meaningful_offset, server.second_replid_offset, server.repl_backlog != NULL, server.repl_backlog_size, @@ -4783,6 +4786,7 @@ void loadDataFromDisk(void) { { memcpy(server.replid,rsi.repl_id,sizeof(server.replid)); server.master_repl_offset = rsi.repl_offset; + server.master_repl_meaningful_offset = rsi.repl_offset; /* If we are a slave, create a cached master from this * information, in order to allow partial resynchronizations * with masters. */ diff --git a/src/server.h b/src/server.h index ed4707d66..818fbc3b3 100644 --- a/src/server.h +++ b/src/server.h @@ -1241,6 +1241,7 @@ struct redisServer { char replid[CONFIG_RUN_ID_SIZE+1]; /* My current replication ID. */ char replid2[CONFIG_RUN_ID_SIZE+1]; /* replid inherited from master*/ long long master_repl_offset; /* My current replication offset */ + long long master_repl_meaningful_offset; /* Offset minus latest PINGs. */ long long second_replid_offset; /* Accept offsets up to this for replid2. */ int slaveseldb; /* Last SELECTed DB in replication output */ int repl_ping_slave_period; /* Master pings the slave every N seconds */ From e257f121e19f3a4ec1d489bd07d75a21731de778 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 25 Mar 2020 15:43:34 +0100 Subject: [PATCH 172/225] PSYNC2: meaningful offset test. --- tests/integration/psync2-pingoff.tcl | 61 ++++++++++++++++++++++++++++ tests/test_helper.tcl | 1 + 2 files changed, 62 insertions(+) create mode 100644 tests/integration/psync2-pingoff.tcl diff --git a/tests/integration/psync2-pingoff.tcl b/tests/integration/psync2-pingoff.tcl new file mode 100644 index 000000000..1cea290e7 --- /dev/null +++ b/tests/integration/psync2-pingoff.tcl @@ -0,0 +1,61 @@ +# Test the meaningful offset implementation to make sure masters +# are able to PSYNC with replicas even if the replication stream +# has pending PINGs at the end. + +start_server {tags {"psync2"}} { +start_server {} { + # Config + set debug_msg 0 ; # Enable additional debug messages + + for {set j 0} {$j < 2} {incr j} { + set R($j) [srv [expr 0-$j] client] + set R_host($j) [srv [expr 0-$j] host] + set R_port($j) [srv [expr 0-$j] port] + $R($j) CONFIG SET repl-ping-replica-period 1 + if {$debug_msg} {puts "Log file: [srv [expr 0-$j] stdout]"} + } + + # Setup replication + test "PSYNC2 meaningful offset: setup" { + $R(1) replicaof $R_host(0) $R_port(0) + $R(0) set foo bar + wait_for_condition 50 1000 { + [$R(0) dbsize] == 1 && [$R(1) dbsize] == 1 + } else { + fail "Replicas not replicating from master" + } + } + + test "PSYNC2 meaningful offset: write and wait replication" { + $R(0) INCR counter + $R(0) INCR counter + $R(0) INCR counter + wait_for_condition 50 1000 { + [$R(0) GET counter] eq [$R(1) GET counter] + } else { + fail "Master and replica don't agree about counter" + } + } + + # In this test we'll make sure the replica will get stuck, but with + # an active connection: this way the master will continue to send PINGs + # every second (we modified the PING period earlier) + test "PSYNC2 meaningful offset: pause replica and promote it" { + $R(1) MULTI + $R(1) DEBUG SLEEP 5 + $R(1) SLAVEOF NO ONE + $R(1) EXEC + $R(1) ping ; # Wait for it to return back available + } + + test "Make the old master a replica of the new one and check conditions" { + set sync_partial [status $R(1) sync_partial_ok] + assert {$sync_partial == 0} + $R(0) REPLICAOF $R_host(1) $R_port(1) + wait_for_condition 50 1000 { + [status $R(1) sync_partial_ok] == 1 + } else { + fail "The new master was not able to partial sync" + } + } +}} diff --git a/tests/test_helper.tcl b/tests/test_helper.tcl index 4dbead193..5cb43104b 100644 --- a/tests/test_helper.tcl +++ b/tests/test_helper.tcl @@ -47,6 +47,7 @@ set ::all_tests { integration/logging integration/psync2 integration/psync2-reg + integration/psync2-pingoff unit/pubsub unit/slowlog unit/scripting From 11db53f875ad136483c7ba55f6049c72c9c6137b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BC=AF=E6=88=90?= Date: Thu, 1 Mar 2018 11:46:56 +0800 Subject: [PATCH 173/225] Boost up performance for redis PUB-SUB patterns matching If lots of clients PSUBSCRIBE to same patterns, multiple pattens matching will take place. This commit change it into just one single pattern matching by using a `dict *` to store the unique pattern and which clients subscribe to it. --- src/pubsub.c | 52 +++++++++++++++++++++++++++++++++++++++++----------- src/server.c | 1 + src/server.h | 1 + 3 files changed, 43 insertions(+), 11 deletions(-) diff --git a/src/pubsub.c b/src/pubsub.c index 5cb4298e0..6fa397704 100644 --- a/src/pubsub.c +++ b/src/pubsub.c @@ -206,6 +206,8 @@ int pubsubUnsubscribeChannel(client *c, robj *channel, int notify) { /* Subscribe a client to a pattern. Returns 1 if the operation succeeded, or 0 if the client was already subscribed to that pattern. */ int pubsubSubscribePattern(client *c, robj *pattern) { + dictEntry *de; + list *clients; int retval = 0; if (listSearchKey(c->pubsub_patterns,pattern) == NULL) { @@ -217,6 +219,16 @@ int pubsubSubscribePattern(client *c, robj *pattern) { pat->pattern = getDecodedObject(pattern); pat->client = c; listAddNodeTail(server.pubsub_patterns,pat); + /* Add the client to the pattern -> list of clients hash table */ + de = dictFind(server.pubsub_patterns_dict,pattern); + if (de == NULL) { + clients = listCreate(); + dictAdd(server.pubsub_patterns_dict,pattern,clients); + incrRefCount(pattern); + } else { + clients = dictGetVal(de); + } + listAddNodeTail(clients,c); } /* Notify the client */ addReplyPubsubPatSubscribed(c,pattern); @@ -226,6 +238,8 @@ int pubsubSubscribePattern(client *c, robj *pattern) { /* Unsubscribe a client from a channel. Returns 1 if the operation succeeded, or * 0 if the client was not subscribed to the specified channel. */ int pubsubUnsubscribePattern(client *c, robj *pattern, int notify) { + dictEntry *de; + list *clients; listNode *ln; pubsubPattern pat; int retval = 0; @@ -238,6 +252,18 @@ int pubsubUnsubscribePattern(client *c, robj *pattern, int notify) { pat.pattern = pattern; ln = listSearchKey(server.pubsub_patterns,&pat); listDelNode(server.pubsub_patterns,ln); + /* Remove the client from the pattern -> clients list hash table */ + de = dictFind(server.pubsub_patterns_dict,pattern); + serverAssertWithInfo(c,NULL,de != NULL); + clients = dictGetVal(de); + ln = listSearchKey(clients,c); + serverAssertWithInfo(c,NULL,ln != NULL); + listDelNode(clients,ln); + if (listLength(clients) == 0) { + /* Free the list and associated hash entry at all if this was + * the latest client. */ + dictDelete(server.pubsub_patterns_dict,pattern); + } } /* Notify the client */ if (notify) addReplyPubsubPatUnsubscribed(c,pattern); @@ -284,6 +310,7 @@ int pubsubUnsubscribeAllPatterns(client *c, int notify) { int pubsubPublishMessage(robj *channel, robj *message) { int receivers = 0; dictEntry *de; + dictIterator *di; listNode *ln; listIter li; @@ -302,23 +329,26 @@ int pubsubPublishMessage(robj *channel, robj *message) { } } /* Send to clients listening to matching channels */ - if (listLength(server.pubsub_patterns)) { - listRewind(server.pubsub_patterns,&li); + di = dictGetIterator(server.pubsub_patterns_dict); + if (di) { channel = getDecodedObject(channel); - while ((ln = listNext(&li)) != NULL) { - pubsubPattern *pat = ln->value; - - if (stringmatchlen((char*)pat->pattern->ptr, - sdslen(pat->pattern->ptr), + while((de = dictNext(di)) != NULL) { + robj *pattern = dictGetKey(de); + list *clients = dictGetVal(de); + if (!stringmatchlen((char*)pattern->ptr, + sdslen(pattern->ptr), (char*)channel->ptr, - sdslen(channel->ptr),0)) - { - addReplyPubsubPatMessage(pat->client, - pat->pattern,channel,message); + sdslen(channel->ptr),0)) continue; + + listRewind(clients,&li); + while ((ln = listNext(&li)) != NULL) { + client *c = listNodeValue(ln); + addReplyPubsubPatMessage(c,pattern,channel,message); receivers++; } } decrRefCount(channel); + dictReleaseIterator(di); } return receivers; } diff --git a/src/server.c b/src/server.c index 84439461e..861aea631 100644 --- a/src/server.c +++ b/src/server.c @@ -2787,6 +2787,7 @@ void initServer(void) { evictionPoolAlloc(); /* Initialize the LRU keys pool. */ server.pubsub_channels = dictCreate(&keylistDictType,NULL); server.pubsub_patterns = listCreate(); + server.pubsub_patterns_dict = dictCreate(&keylistDictType,NULL); listSetFreeMethod(server.pubsub_patterns,freePubsubPattern); listSetMatchMethod(server.pubsub_patterns,listMatchPubsubPattern); server.cronloops = 0; diff --git a/src/server.h b/src/server.h index 818fbc3b3..cb2f864a1 100644 --- a/src/server.h +++ b/src/server.h @@ -1344,6 +1344,7 @@ struct redisServer { /* Pubsub */ dict *pubsub_channels; /* Map channels to list of subscribed clients */ list *pubsub_patterns; /* A list of pubsub_patterns */ + dict *pubsub_patterns_dict; /* A dict of pubsub_patterns */ int notify_keyspace_events; /* Events to propagate via Pub/Sub. This is an xor of NOTIFY_... flags. */ /* Cluster */ From 316a8f154530cb49c5beaa26757805a911a2e667 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 27 Mar 2020 16:19:56 +0100 Subject: [PATCH 174/225] PSYNC2: fix backlog_idx when adjusting for meaningful offset See #7002. --- src/replication.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/replication.c b/src/replication.c index e62afb2fb..20f9a2129 100644 --- a/src/replication.c +++ b/src/replication.c @@ -2755,6 +2755,9 @@ void replicationCacheMasterUsingMyself(void) { delta); server.master_initial_offset = server.master_repl_meaningful_offset; server.repl_backlog_histlen -= delta; + server.repl_backlog_idx = + (server.repl_backlog_idx + (server.repl_backlog_size - delta)) % + server.repl_backlog_size; if (server.repl_backlog_histlen < 0) server.repl_backlog_histlen = 0; } From 9d6d1779944c6d3847cc0fe9d4066a6cc1e95f35 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 26 Mar 2020 11:33:18 +0100 Subject: [PATCH 175/225] Precise timeouts: refactor unblocking on timeout. --- src/server.c | 41 +++++++++++++++++++++++++++++------------ src/server.h | 3 +++ 2 files changed, 32 insertions(+), 12 deletions(-) diff --git a/src/server.c b/src/server.c index 861aea631..e657f1196 100644 --- a/src/server.c +++ b/src/server.c @@ -1502,6 +1502,23 @@ long long getInstantaneousMetric(int metric) { return sum / STATS_METRIC_SAMPLES; } +/* Check if this blocked client timedout (does nothing if the client is + * not blocked right now). If so send a reply, unblock it, and return 1. + * Otherwise 0 is returned and no operation is performed. */ +int checkBlockedClientTimeout(client *c, mstime_t now) { + if (c->flags & CLIENT_BLOCKED && + c->bpop.timeout != 0 + && c->bpop.timeout < now) + { + /* Handle blocking operation specific timeout. */ + replyToBlockedClientTimedOut(c); + unblockClient(c); + return 1; + } else { + return 0; + } +} + /* Check for timeouts. Returns non-zero if the client was terminated. * The function gets the current time in milliseconds as argument since * it gets called multiple times in a loop, so calling gettimeofday() for @@ -1510,10 +1527,11 @@ int clientsCronHandleTimeout(client *c, mstime_t now_ms) { time_t now = now_ms/1000; if (server.maxidletime && - !(c->flags & CLIENT_SLAVE) && /* no timeout for slaves and monitors */ - !(c->flags & CLIENT_MASTER) && /* no timeout for masters */ - !(c->flags & CLIENT_BLOCKED) && /* no timeout for BLPOP */ - !(c->flags & CLIENT_PUBSUB) && /* no timeout for Pub/Sub clients */ + /* This handles the idle clients connection timeout if set. */ + !(c->flags & CLIENT_SLAVE) && /* No timeout for slaves and monitors */ + !(c->flags & CLIENT_MASTER) && /* No timeout for masters */ + !(c->flags & CLIENT_BLOCKED) && /* No timeout for BLPOP */ + !(c->flags & CLIENT_PUBSUB) && /* No timeout for Pub/Sub clients */ (now - c->lastinteraction > server.maxidletime)) { serverLog(LL_VERBOSE,"Closing idle client"); @@ -1522,15 +1540,14 @@ int clientsCronHandleTimeout(client *c, mstime_t now_ms) { } else if (c->flags & CLIENT_BLOCKED) { /* Blocked OPS timeout is handled with milliseconds resolution. * However note that the actual resolution is limited by - * server.hz. */ + * server.hz. So for short timeouts (less than SERVER_SHORT_TIMEOUT + * milliseconds) we populate a Radix tree and handle such timeouts + * in clientsHandleShortTimeout(). */ + if (checkBlockedClientTimeout(c,now_ms)) return 0; - if (c->bpop.timeout != 0 && c->bpop.timeout < now_ms) { - /* Handle blocking operation specific timeout. */ - replyToBlockedClientTimedOut(c); - unblockClient(c); - } else if (server.cluster_enabled) { - /* Cluster: handle unblock & redirect of clients blocked - * into keys no longer served by this server. */ + /* Cluster: handle unblock & redirect of clients blocked + * into keys no longer served by this server. */ + if (server.cluster_enabled) { if (clusterRedirectBlockedClientIfNeeded(c)) unblockClient(c); } diff --git a/src/server.h b/src/server.h index cb2f864a1..d697e79a2 100644 --- a/src/server.h +++ b/src/server.h @@ -277,6 +277,9 @@ typedef long long ustime_t; /* microsecond time type. */ buffer configuration. Just the first three: normal, slave, pubsub. */ +/* Other client related defines. */ +#define CLIENT_SHORT_TIMEOUT 2000 /* See clientsHandleShortTimeout(). */ + /* Slave replication state. Used in server.repl_state for slaves to remember * what to do next. */ #define REPL_STATE_NONE 0 /* No active replication */ From 7add0f249078aa11cdeb11ec7535519315f0369e Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 26 Mar 2020 13:28:39 +0100 Subject: [PATCH 176/225] Precise timeouts: working initial implementation. --- src/blocked.c | 1 + src/server.c | 135 +++++++++++++++++++++++++++++++++++++++----------- src/server.h | 2 + 3 files changed, 110 insertions(+), 28 deletions(-) diff --git a/src/blocked.c b/src/blocked.c index 06aa5850e..c470cba00 100644 --- a/src/blocked.c +++ b/src/blocked.c @@ -619,6 +619,7 @@ void blockForKeys(client *c, int btype, robj **keys, int numkeys, mstime_t timeo listAddNodeTail(l,c); } blockClient(c,btype); + addClientToShortTimeoutTable(c); } /* Unblock a client that's waiting in a blocking operation such as BLPOP. diff --git a/src/server.c b/src/server.c index e657f1196..26581d979 100644 --- a/src/server.c +++ b/src/server.c @@ -1473,34 +1473,7 @@ int allPersistenceDisabled(void) { return server.saveparamslen == 0 && server.aof_state == AOF_OFF; } -/* ======================= Cron: called every 100 ms ======================== */ - -/* Add a sample to the operations per second array of samples. */ -void trackInstantaneousMetric(int metric, long long current_reading) { - long long t = mstime() - server.inst_metric[metric].last_sample_time; - long long ops = current_reading - - server.inst_metric[metric].last_sample_count; - long long ops_sec; - - ops_sec = t > 0 ? (ops*1000/t) : 0; - - server.inst_metric[metric].samples[server.inst_metric[metric].idx] = - ops_sec; - server.inst_metric[metric].idx++; - server.inst_metric[metric].idx %= STATS_METRIC_SAMPLES; - server.inst_metric[metric].last_sample_time = mstime(); - server.inst_metric[metric].last_sample_count = current_reading; -} - -/* Return the mean of all the samples. */ -long long getInstantaneousMetric(int metric) { - int j; - long long sum = 0; - - for (j = 0; j < STATS_METRIC_SAMPLES; j++) - sum += server.inst_metric[metric].samples[j]; - return sum / STATS_METRIC_SAMPLES; -} +/* ========================== Clients timeouts ============================= */ /* Check if this blocked client timedout (does nothing if the client is * not blocked right now). If so send a reply, unblock it, and return 1. @@ -1555,6 +1528,107 @@ int clientsCronHandleTimeout(client *c, mstime_t now_ms) { return 0; } +/* For shor timeouts, less than < CLIENT_SHORT_TIMEOUT milliseconds, we + * populate a radix tree of 128 bit keys composed as such: + * + * [8 byte big endian expire time]+[8 byte client ID] + * + * We don't do any cleanup in the Radix tree: when we run the clients that + * reached the timeout already, if they are no longer existing or no longer + * blocked with such timeout, we just go forward. + * + * Every time a client blocks with a short timeout, we add the client in + * the tree. In beforeSleep() we call clientsHandleShortTimeout() to run + * the tree and unblock the clients. + * + * Design hint: why we block only clients with short timeouts? For frugality: + * Clients blocking for 30 seconds usually don't need to be unblocked + * precisely, and anyway for the nature of Redis to *guarantee* unblock time + * precision is hard, so we can avoid putting a large number of clients in + * the radix tree without a good reason. This idea also has a role in memory + * usage as well given that we don't do cleanup, the shorter a client timeout, + * the less time it will stay in the radix tree. */ + +#define CLIENT_ST_KEYLEN 16 /* 8 bytes mstime + 8 bytes client ID. */ + +/* Given client ID and timeout, write the resulting radix tree key in buf. */ +void encodeTimeoutKey(unsigned char *buf, uint64_t timeout, uint64_t id) { + timeout = htonu64(timeout); + memcpy(buf,&timeout,sizeof(timeout)); + memcpy(buf+8,&id,sizeof(id)); +} + +/* Given a key encoded with encodeTimeoutKey(), resolve the fields and write + * the timeout into *toptr and the client ID into *idptr. */ +void decodeTimeoutKey(unsigned char *buf, uint64_t *toptr, uint64_t *idptr) { + memcpy(toptr,buf,sizeof(*toptr)); + *toptr = ntohu64(*toptr); + memcpy(idptr,buf+8,sizeof(*idptr)); +} + +/* Add the specified client id / timeout as a key in the radix tree we use + * to handle short timeouts. The client is not added to the list if its + * timeout is longer than CLIENT_SHORT_TIMEOUT milliseconds. */ +void addClientToShortTimeoutTable(client *c) { + if (c->bpop.timeout == 0 || + c->bpop.timeout - mstime() > CLIENT_SHORT_TIMEOUT) + { + return; + } + uint64_t timeout = c->bpop.timeout; + uint64_t id = c->id; + unsigned char buf[CLIENT_ST_KEYLEN]; + encodeTimeoutKey(buf,timeout,id); + raxTryInsert(server.clients_timeout_table,buf,sizeof(buf),NULL,NULL); +} + +/* This function is called in beforeSleep() in order to unblock ASAP clients + * that are waiting in blocking operations with a short timeout set. */ +void clientsHandleShortTimeout(void) { + uint64_t now = mstime(); + raxIterator ri; + raxStart(&ri,server.clients_timeout_table); + + while(raxNext(&ri)) { + uint64_t id, timeout; + decodeTimeoutKey(ri.key,&timeout,&id); + if (timeout >= now) break; /* All the timeouts are in the future. */ + client *c = lookupClientByID(id); + if (c) checkBlockedClientTimeout(c,now); + raxRemove(server.clients_timeout_table,ri.key,ri.key_len,NULL); + raxSeek(&ri,"^",NULL,0); + } +} + +/* ======================= Cron: called every 100 ms ======================== */ + +/* Add a sample to the operations per second array of samples. */ +void trackInstantaneousMetric(int metric, long long current_reading) { + long long t = mstime() - server.inst_metric[metric].last_sample_time; + long long ops = current_reading - + server.inst_metric[metric].last_sample_count; + long long ops_sec; + + ops_sec = t > 0 ? (ops*1000/t) : 0; + + server.inst_metric[metric].samples[server.inst_metric[metric].idx] = + ops_sec; + server.inst_metric[metric].idx++; + server.inst_metric[metric].idx %= STATS_METRIC_SAMPLES; + server.inst_metric[metric].last_sample_time = mstime(); + server.inst_metric[metric].last_sample_count = current_reading; +} + +/* Return the mean of all the samples. */ +long long getInstantaneousMetric(int metric) { + int j; + long long sum = 0; + + for (j = 0; j < STATS_METRIC_SAMPLES; j++) + sum += server.inst_metric[metric].samples[j]; + return sum / STATS_METRIC_SAMPLES; +} + /* The client query buffer is an sds.c string that can end with a lot of * free space not used, this function reclaims space if needed. * @@ -2109,11 +2183,15 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) { void beforeSleep(struct aeEventLoop *eventLoop) { UNUSED(eventLoop); + /* Handle precise timeouts of blocked clients. */ + clientsHandleShortTimeout(); + /* We should handle pending reads clients ASAP after event loop. */ handleClientsWithPendingReadsUsingThreads(); /* Handle TLS pending data. (must be done before flushAppendOnlyFile) */ tlsProcessPendingData(); + /* If tls still has pending unread data don't sleep at all. */ aeSetDontWait(server.el, tlsHasPendingData()); @@ -2738,6 +2816,7 @@ void initServer(void) { server.monitors = listCreate(); server.clients_pending_write = listCreate(); server.clients_pending_read = listCreate(); + server.clients_timeout_table = raxNew(); server.slaveseldb = -1; /* Force to emit the first SELECT command. */ server.unblocked_clients = listCreate(); server.ready_keys = listCreate(); diff --git a/src/server.h b/src/server.h index d697e79a2..c20f70136 100644 --- a/src/server.h +++ b/src/server.h @@ -1070,6 +1070,7 @@ struct redisServer { list *clients_pending_read; /* Client has pending read socket buffers. */ list *slaves, *monitors; /* List of slaves and MONITORs */ client *current_client; /* Current client executing the command. */ + rax *clients_timeout_table; /* Radix tree for clients with short timeout. */ long fixed_time_expire; /* If > 0, expire keys against server.mstime. */ rax *clients_index; /* Active clients dictionary by client ID. */ int clients_paused; /* True if clients are currently paused */ @@ -2140,6 +2141,7 @@ void disconnectAllBlockedClients(void); void handleClientsBlockedOnKeys(void); void signalKeyAsReady(redisDb *db, robj *key); void blockForKeys(client *c, int btype, robj **keys, int numkeys, mstime_t timeout, robj *target, streamID *ids); +void addClientToShortTimeoutTable(client *c); /* expire.c -- Handling of expired keys */ void activeExpireCycle(int type); From 30f1df8c48406b21b3ec896f652b25a484f4dd46 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 26 Mar 2020 14:37:00 +0100 Subject: [PATCH 177/225] Precise timeouts: fix bugs in initial implementation. --- src/blocked.c | 2 +- src/server.c | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/blocked.c b/src/blocked.c index c470cba00..84a5287d4 100644 --- a/src/blocked.c +++ b/src/blocked.c @@ -111,6 +111,7 @@ void blockClient(client *c, int btype) { c->btype = btype; server.blocked_clients++; server.blocked_clients_by_type[btype]++; + addClientToShortTimeoutTable(c); } /* This function is called in the beforeSleep() function of the event loop @@ -619,7 +620,6 @@ void blockForKeys(client *c, int btype, robj **keys, int numkeys, mstime_t timeo listAddNodeTail(l,c); } blockClient(c,btype); - addClientToShortTimeoutTable(c); } /* Unblock a client that's waiting in a blocking operation such as BLPOP. diff --git a/src/server.c b/src/server.c index 26581d979..19f43e413 100644 --- a/src/server.c +++ b/src/server.c @@ -1588,6 +1588,7 @@ void clientsHandleShortTimeout(void) { uint64_t now = mstime(); raxIterator ri; raxStart(&ri,server.clients_timeout_table); + raxSeek(&ri,"^",NULL,0); while(raxNext(&ri)) { uint64_t id, timeout; @@ -1745,6 +1746,9 @@ void getExpansiveClientsInfo(size_t *in_usage, size_t *out_usage) { */ #define CLIENTS_CRON_MIN_ITERATIONS 5 void clientsCron(void) { + /* Unblock short timeout clients ASAP. */ + clientsHandleShortTimeout(); + /* Try to process at least numclients/server.hz of clients * per call. Since normally (if there are no big latency events) this * function is called server.hz times per second, in the average case we From 6862fd70da38b0b09d08a307924f2e7b6bcf297a Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 26 Mar 2020 15:52:16 +0100 Subject: [PATCH 178/225] Precise timeouts: fast exit for clientsHandleShortTimeout(). --- src/server.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/server.c b/src/server.c index 19f43e413..9a86ae3de 100644 --- a/src/server.c +++ b/src/server.c @@ -1585,6 +1585,7 @@ void addClientToShortTimeoutTable(client *c) { /* This function is called in beforeSleep() in order to unblock ASAP clients * that are waiting in blocking operations with a short timeout set. */ void clientsHandleShortTimeout(void) { + if (raxSize(server.clients_timeout_table) == 0) return; uint64_t now = mstime(); raxIterator ri; raxStart(&ri,server.clients_timeout_table); From a443ec2e9a7a34e822ce41febb7b84ee4ee3c45c Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 26 Mar 2020 16:02:26 +0100 Subject: [PATCH 179/225] Precise timeouts: use only radix tree for timeouts. --- src/blocked.c | 2 +- src/server.c | 46 +++++++++++++--------------------------------- src/server.h | 5 +---- 3 files changed, 15 insertions(+), 38 deletions(-) diff --git a/src/blocked.c b/src/blocked.c index 84a5287d4..443daec7f 100644 --- a/src/blocked.c +++ b/src/blocked.c @@ -111,7 +111,7 @@ void blockClient(client *c, int btype) { c->btype = btype; server.blocked_clients++; server.blocked_clients_by_type[btype]++; - addClientToShortTimeoutTable(c); + addClientToTimeoutTable(c); } /* This function is called in the beforeSleep() function of the event loop diff --git a/src/server.c b/src/server.c index 9a86ae3de..f14ecc0cc 100644 --- a/src/server.c +++ b/src/server.c @@ -1511,13 +1511,6 @@ int clientsCronHandleTimeout(client *c, mstime_t now_ms) { freeClient(c); return 1; } else if (c->flags & CLIENT_BLOCKED) { - /* Blocked OPS timeout is handled with milliseconds resolution. - * However note that the actual resolution is limited by - * server.hz. So for short timeouts (less than SERVER_SHORT_TIMEOUT - * milliseconds) we populate a Radix tree and handle such timeouts - * in clientsHandleShortTimeout(). */ - if (checkBlockedClientTimeout(c,now_ms)) return 0; - /* Cluster: handle unblock & redirect of clients blocked * into keys no longer served by this server. */ if (server.cluster_enabled) { @@ -1528,8 +1521,8 @@ int clientsCronHandleTimeout(client *c, mstime_t now_ms) { return 0; } -/* For shor timeouts, less than < CLIENT_SHORT_TIMEOUT milliseconds, we - * populate a radix tree of 128 bit keys composed as such: +/* For blocked clients timeouts we populate a radix tree of 128 bit keys + * composed as such: * * [8 byte big endian expire time]+[8 byte client ID] * @@ -1538,16 +1531,8 @@ int clientsCronHandleTimeout(client *c, mstime_t now_ms) { * blocked with such timeout, we just go forward. * * Every time a client blocks with a short timeout, we add the client in - * the tree. In beforeSleep() we call clientsHandleShortTimeout() to run - * the tree and unblock the clients. - * - * Design hint: why we block only clients with short timeouts? For frugality: - * Clients blocking for 30 seconds usually don't need to be unblocked - * precisely, and anyway for the nature of Redis to *guarantee* unblock time - * precision is hard, so we can avoid putting a large number of clients in - * the radix tree without a good reason. This idea also has a role in memory - * usage as well given that we don't do cleanup, the shorter a client timeout, - * the less time it will stay in the radix tree. */ + * the tree. In beforeSleep() we call clientsHandleTimeout() to run + * the tree and unblock the clients. */ #define CLIENT_ST_KEYLEN 16 /* 8 bytes mstime + 8 bytes client ID. */ @@ -1568,13 +1553,9 @@ void decodeTimeoutKey(unsigned char *buf, uint64_t *toptr, uint64_t *idptr) { /* Add the specified client id / timeout as a key in the radix tree we use * to handle short timeouts. The client is not added to the list if its - * timeout is longer than CLIENT_SHORT_TIMEOUT milliseconds. */ -void addClientToShortTimeoutTable(client *c) { - if (c->bpop.timeout == 0 || - c->bpop.timeout - mstime() > CLIENT_SHORT_TIMEOUT) - { - return; - } + * timeout is zero (block forever). */ +void addClientToTimeoutTable(client *c) { + if (c->bpop.timeout == 0) return; uint64_t timeout = c->bpop.timeout; uint64_t id = c->id; unsigned char buf[CLIENT_ST_KEYLEN]; @@ -1584,7 +1565,7 @@ void addClientToShortTimeoutTable(client *c) { /* This function is called in beforeSleep() in order to unblock ASAP clients * that are waiting in blocking operations with a short timeout set. */ -void clientsHandleShortTimeout(void) { +void clientsHandleTimeout(void) { if (raxSize(server.clients_timeout_table) == 0) return; uint64_t now = mstime(); raxIterator ri; @@ -1747,9 +1728,6 @@ void getExpansiveClientsInfo(size_t *in_usage, size_t *out_usage) { */ #define CLIENTS_CRON_MIN_ITERATIONS 5 void clientsCron(void) { - /* Unblock short timeout clients ASAP. */ - clientsHandleShortTimeout(); - /* Try to process at least numclients/server.hz of clients * per call. Since normally (if there are no big latency events) this * function is called server.hz times per second, in the average case we @@ -2189,7 +2167,7 @@ void beforeSleep(struct aeEventLoop *eventLoop) { UNUSED(eventLoop); /* Handle precise timeouts of blocked clients. */ - clientsHandleShortTimeout(); + clientsHandleTimeout(); /* We should handle pending reads clients ASAP after event loop. */ handleClientsWithPendingReadsUsingThreads(); @@ -4102,11 +4080,13 @@ sds genRedisInfoString(const char *section) { "client_recent_max_input_buffer:%zu\r\n" "client_recent_max_output_buffer:%zu\r\n" "blocked_clients:%d\r\n" - "tracking_clients:%d\r\n", + "tracking_clients:%d\r\n" + "clients_in_timeout_table:%lld\r\n", listLength(server.clients)-listLength(server.slaves), maxin, maxout, server.blocked_clients, - server.tracking_clients); + server.tracking_clients, + raxSize(server.clients_timeout_table)); } /* Memory */ diff --git a/src/server.h b/src/server.h index c20f70136..4801a5aed 100644 --- a/src/server.h +++ b/src/server.h @@ -277,9 +277,6 @@ typedef long long ustime_t; /* microsecond time type. */ buffer configuration. Just the first three: normal, slave, pubsub. */ -/* Other client related defines. */ -#define CLIENT_SHORT_TIMEOUT 2000 /* See clientsHandleShortTimeout(). */ - /* Slave replication state. Used in server.repl_state for slaves to remember * what to do next. */ #define REPL_STATE_NONE 0 /* No active replication */ @@ -2141,7 +2138,7 @@ void disconnectAllBlockedClients(void); void handleClientsBlockedOnKeys(void); void signalKeyAsReady(redisDb *db, robj *key); void blockForKeys(client *c, int btype, robj **keys, int numkeys, mstime_t timeout, robj *target, streamID *ids); -void addClientToShortTimeoutTable(client *c); +void addClientToTimeoutTable(client *c); /* expire.c -- Handling of expired keys */ void activeExpireCycle(int type); From ad94066ec8b2c665aa4d5f49d79d9de3f95428b0 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 26 Mar 2020 16:05:20 +0100 Subject: [PATCH 180/225] Precise timeouts: fix comments after functional change. --- src/server.c | 10 +++++----- src/server.h | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/server.c b/src/server.c index f14ecc0cc..cb3c143f8 100644 --- a/src/server.c +++ b/src/server.c @@ -1530,7 +1530,7 @@ int clientsCronHandleTimeout(client *c, mstime_t now_ms) { * reached the timeout already, if they are no longer existing or no longer * blocked with such timeout, we just go forward. * - * Every time a client blocks with a short timeout, we add the client in + * Every time a client blocks with a timeout, we add the client in * the tree. In beforeSleep() we call clientsHandleTimeout() to run * the tree and unblock the clients. */ @@ -1552,8 +1552,8 @@ void decodeTimeoutKey(unsigned char *buf, uint64_t *toptr, uint64_t *idptr) { } /* Add the specified client id / timeout as a key in the radix tree we use - * to handle short timeouts. The client is not added to the list if its - * timeout is zero (block forever). */ + * to handle blocked clients timeouts. The client is not added to the list + * if its timeout is zero (block forever). */ void addClientToTimeoutTable(client *c) { if (c->bpop.timeout == 0) return; uint64_t timeout = c->bpop.timeout; @@ -1563,8 +1563,8 @@ void addClientToTimeoutTable(client *c) { raxTryInsert(server.clients_timeout_table,buf,sizeof(buf),NULL,NULL); } -/* This function is called in beforeSleep() in order to unblock ASAP clients - * that are waiting in blocking operations with a short timeout set. */ +/* This function is called in beforeSleep() in order to unblock clients + * that are waiting in blocking operations with a timeout set. */ void clientsHandleTimeout(void) { if (raxSize(server.clients_timeout_table) == 0) return; uint64_t now = mstime(); diff --git a/src/server.h b/src/server.h index 4801a5aed..4a8426bab 100644 --- a/src/server.h +++ b/src/server.h @@ -1067,7 +1067,7 @@ struct redisServer { list *clients_pending_read; /* Client has pending read socket buffers. */ list *slaves, *monitors; /* List of slaves and MONITORs */ client *current_client; /* Current client executing the command. */ - rax *clients_timeout_table; /* Radix tree for clients with short timeout. */ + rax *clients_timeout_table; /* Radix tree for blocked clients timeouts. */ long fixed_time_expire; /* If > 0, expire keys against server.mstime. */ rax *clients_index; /* Active clients dictionary by client ID. */ int clients_paused; /* True if clients are currently paused */ From 67643ead69ffd0e3d6a4d6fbd7835ec90beb7bd3 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 27 Mar 2020 11:13:38 +0100 Subject: [PATCH 181/225] Precise timeouts: cleaup the table on unblock. Now that this mechanism is the sole one used for blocked clients timeouts, it is more wise to cleanup the table when the client unblocks for any reason. We use a flag: CLIENT_IN_TO_TABLE, in order to avoid a radix tree lookup when the client was already removed from the table because we processed it by scanning the radix tree. --- src/blocked.c | 1 + src/server.c | 20 ++++++++++++++++++-- src/server.h | 2 ++ 3 files changed, 21 insertions(+), 2 deletions(-) diff --git a/src/blocked.c b/src/blocked.c index 443daec7f..795985ea1 100644 --- a/src/blocked.c +++ b/src/blocked.c @@ -186,6 +186,7 @@ void unblockClient(client *c) { server.blocked_clients_by_type[c->btype]--; c->flags &= ~CLIENT_BLOCKED; c->btype = BLOCKED_NONE; + removeClientFromTimeoutTable(c); queueClientForReprocessing(c); } diff --git a/src/server.c b/src/server.c index cb3c143f8..9a89290ca 100644 --- a/src/server.c +++ b/src/server.c @@ -1560,7 +1560,20 @@ void addClientToTimeoutTable(client *c) { uint64_t id = c->id; unsigned char buf[CLIENT_ST_KEYLEN]; encodeTimeoutKey(buf,timeout,id); - raxTryInsert(server.clients_timeout_table,buf,sizeof(buf),NULL,NULL); + if (raxTryInsert(server.clients_timeout_table,buf,sizeof(buf),NULL,NULL)) + c->flags |= CLIENT_IN_TO_TABLE; +} + +/* Remove the client from the table when it is unblocked for reasons + * different than timing out. */ +void removeClientFromTimeoutTable(client *c) { + if (!(c->flags & CLIENT_IN_TO_TABLE)) return; + c->flags &= ~CLIENT_IN_TO_TABLE; + uint64_t timeout = c->bpop.timeout; + uint64_t id = c->id; + unsigned char buf[CLIENT_ST_KEYLEN]; + encodeTimeoutKey(buf,timeout,id); + raxRemove(server.clients_timeout_table,buf,sizeof(buf),NULL); } /* This function is called in beforeSleep() in order to unblock clients @@ -1577,7 +1590,10 @@ void clientsHandleTimeout(void) { decodeTimeoutKey(ri.key,&timeout,&id); if (timeout >= now) break; /* All the timeouts are in the future. */ client *c = lookupClientByID(id); - if (c) checkBlockedClientTimeout(c,now); + if (c) { + c->flags &= ~CLIENT_IN_TO_TABLE; + checkBlockedClientTimeout(c,now); + } raxRemove(server.clients_timeout_table,ri.key,ri.key_len,NULL); raxSeek(&ri,"^",NULL,0); } diff --git a/src/server.h b/src/server.h index 4a8426bab..03a153c7b 100644 --- a/src/server.h +++ b/src/server.h @@ -252,6 +252,7 @@ typedef long long ustime_t; /* microsecond time type. */ #define CLIENT_TRACKING_OPTOUT (1ULL<<35) /* Tracking in opt-out mode. */ #define CLIENT_TRACKING_CACHING (1ULL<<36) /* CACHING yes/no was given, depending on optin/optout mode. */ +#define CLIENT_IN_TO_TABLE (1ULL<<37) /* This client is in the timeout table. */ /* Client block type (btype field in client structure) * if CLIENT_BLOCKED flag is set. */ @@ -2139,6 +2140,7 @@ void handleClientsBlockedOnKeys(void); void signalKeyAsReady(redisDb *db, robj *key); void blockForKeys(client *c, int btype, robj **keys, int numkeys, mstime_t timeout, robj *target, streamID *ids); void addClientToTimeoutTable(client *c); +void removeClientFromTimeoutTable(client *c); /* expire.c -- Handling of expired keys */ void activeExpireCycle(int type); From bbbc80acae773c268919c9bc475f695e3b75dcfd Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 30 Mar 2020 15:22:55 +0200 Subject: [PATCH 182/225] Precise timeouts: reference client pointer directly. --- src/timeout.c | 29 +++++++++++++---------------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/src/timeout.c b/src/timeout.c index ea2032e2a..bb5999418 100644 --- a/src/timeout.c +++ b/src/timeout.c @@ -93,18 +93,19 @@ int clientsCronHandleTimeout(client *c, mstime_t now_ms) { #define CLIENT_ST_KEYLEN 16 /* 8 bytes mstime + 8 bytes client ID. */ /* Given client ID and timeout, write the resulting radix tree key in buf. */ -void encodeTimeoutKey(unsigned char *buf, uint64_t timeout, uint64_t id) { +void encodeTimeoutKey(unsigned char *buf, uint64_t timeout, client *c) { timeout = htonu64(timeout); memcpy(buf,&timeout,sizeof(timeout)); - memcpy(buf+8,&id,sizeof(id)); + memcpy(buf+8,&c,sizeof(c)); + if (sizeof(c) == 4) memset(buf+12,0,4); /* Zero padding for 32bit target. */ } /* Given a key encoded with encodeTimeoutKey(), resolve the fields and write - * the timeout into *toptr and the client ID into *idptr. */ -void decodeTimeoutKey(unsigned char *buf, uint64_t *toptr, uint64_t *idptr) { + * the timeout into *toptr and the client pointer into *cptr. */ +void decodeTimeoutKey(unsigned char *buf, uint64_t *toptr, client **cptr) { memcpy(toptr,buf,sizeof(*toptr)); *toptr = ntohu64(*toptr); - memcpy(idptr,buf+8,sizeof(*idptr)); + memcpy(cptr,buf+8,sizeof(*cptr)); } /* Add the specified client id / timeout as a key in the radix tree we use @@ -113,9 +114,8 @@ void decodeTimeoutKey(unsigned char *buf, uint64_t *toptr, uint64_t *idptr) { void addClientToTimeoutTable(client *c) { if (c->bpop.timeout == 0) return; uint64_t timeout = c->bpop.timeout; - uint64_t id = c->id; unsigned char buf[CLIENT_ST_KEYLEN]; - encodeTimeoutKey(buf,timeout,id); + encodeTimeoutKey(buf,timeout,c); if (raxTryInsert(server.clients_timeout_table,buf,sizeof(buf),NULL,NULL)) c->flags |= CLIENT_IN_TO_TABLE; } @@ -126,9 +126,8 @@ void removeClientFromTimeoutTable(client *c) { if (!(c->flags & CLIENT_IN_TO_TABLE)) return; c->flags &= ~CLIENT_IN_TO_TABLE; uint64_t timeout = c->bpop.timeout; - uint64_t id = c->id; unsigned char buf[CLIENT_ST_KEYLEN]; - encodeTimeoutKey(buf,timeout,id); + encodeTimeoutKey(buf,timeout,c); raxRemove(server.clients_timeout_table,buf,sizeof(buf),NULL); } @@ -142,14 +141,12 @@ void handleBlockedClientsTimeout(void) { raxSeek(&ri,"^",NULL,0); while(raxNext(&ri)) { - uint64_t id, timeout; - decodeTimeoutKey(ri.key,&timeout,&id); + uint64_t timeout; + client *c; + decodeTimeoutKey(ri.key,&timeout,&c); if (timeout >= now) break; /* All the timeouts are in the future. */ - client *c = lookupClientByID(id); - if (c) { - c->flags &= ~CLIENT_IN_TO_TABLE; - checkBlockedClientTimeout(c,now); - } + c->flags &= ~CLIENT_IN_TO_TABLE; + checkBlockedClientTimeout(c,now); raxRemove(server.clients_timeout_table,ri.key,ri.key_len,NULL); raxSeek(&ri,"^",NULL,0); } From fa41863752361f2c9c0627784ecadd65b078b415 Mon Sep 17 00:00:00 2001 From: "zhaozhao.zz" Date: Sat, 28 Mar 2020 20:59:01 +0800 Subject: [PATCH 183/225] PSYNC2: reset backlog_idx and master_repl_offset correctly --- src/replication.c | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/replication.c b/src/replication.c index 20f9a2129..3e9910374 100644 --- a/src/replication.c +++ b/src/replication.c @@ -2754,11 +2754,16 @@ void replicationCacheMasterUsingMyself(void) { server.master_repl_offset, delta); server.master_initial_offset = server.master_repl_meaningful_offset; - server.repl_backlog_histlen -= delta; - server.repl_backlog_idx = - (server.repl_backlog_idx + (server.repl_backlog_size - delta)) % - server.repl_backlog_size; - if (server.repl_backlog_histlen < 0) server.repl_backlog_histlen = 0; + server.master_repl_offset = server.master_repl_meaningful_offset; + if (server.repl_backlog_histlen <= delta) { + server.repl_backlog_histlen = 0; + server.repl_backlog_idx = 0; + } else { + server.repl_backlog_histlen -= delta; + server.repl_backlog_idx = + (server.repl_backlog_idx + (server.repl_backlog_size - delta)) % + server.repl_backlog_size; + } } /* The master client we create can be set to any DBID, because From 26b79ca18c3f4e86d4158c8ad574c3cc2a031652 Mon Sep 17 00:00:00 2001 From: OMG-By <504094596@qq.com> Date: Sun, 29 Mar 2020 00:04:59 +0800 Subject: [PATCH 184/225] fix: dict.c->dictResize()->minimal type --- src/dict.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dict.c b/src/dict.c index 93e6c39a7..ac6f8cfde 100644 --- a/src/dict.c +++ b/src/dict.c @@ -134,7 +134,7 @@ int _dictInit(dict *d, dictType *type, * but with the invariant of a USED/BUCKETS ratio near to <= 1 */ int dictResize(dict *d) { - int minimal; + unsigned long minimal; if (!dict_can_resize || dictIsRehashing(d)) return DICT_ERR; minimal = d->ht[0].used; From 08fdef4bf633fcc76caddae60220f56eec77fd91 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 31 Mar 2020 10:27:47 +0200 Subject: [PATCH 185/225] Fix RM_Call() stale comment due to cut&paste. --- src/module.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/module.c b/src/module.c index 6f61a5ca8..7d9eb3719 100644 --- a/src/module.c +++ b/src/module.c @@ -3341,9 +3341,7 @@ RedisModuleCallReply *RM_Call(RedisModuleCtx *ctx, const char *cmdname, const ch } call(c,call_flags); - /* Convert the result of the Redis command into a suitable Lua type. - * The first thing we need is to create a single string from the client - * output buffers. */ + /* Convert the result of the Redis command into a module reply. */ sds proto = sdsnewlen(c->buf,c->bufpos); c->bufpos = 0; while(listLength(c->reply)) { From 616b1cb7ac543360fcffadce1f762e3de9928511 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 31 Mar 2020 11:00:45 +0200 Subject: [PATCH 186/225] Fix module commands propagation double MULTI bug. 37a10cef introduced automatic wrapping of MULTI/EXEC for the alsoPropagate API. However this collides with the built-in mechanism already present in module.c. To avoid complex changes near Redis 6 GA this commit introduces the ability to exclude call() MUTLI/EXEC wrapping for also propagate in order to continue to use the old code paths in module.c. --- src/module.c | 7 +------ src/server.c | 8 ++++++-- src/server.h | 2 ++ tests/modules/propagate.c | 16 ++++++++++++++++ 4 files changed, 25 insertions(+), 8 deletions(-) diff --git a/src/module.c b/src/module.c index 7d9eb3719..61dc25169 100644 --- a/src/module.c +++ b/src/module.c @@ -3326,13 +3326,8 @@ RedisModuleCallReply *RM_Call(RedisModuleCtx *ctx, const char *cmdname, const ch * a Lua script in the context of AOF and slaves. */ if (replicate) moduleReplicateMultiIfNeeded(ctx); - if (ctx->client->flags & CLIENT_MULTI || - ctx->flags & REDISMODULE_CTX_MULTI_EMITTED) { - c->flags |= CLIENT_MULTI; - } - /* Run the command */ - int call_flags = CMD_CALL_SLOWLOG | CMD_CALL_STATS; + int call_flags = CMD_CALL_SLOWLOG | CMD_CALL_STATS | CMD_CALL_NOWRAP; if (replicate) { if (!(flags & REDISMODULE_ARGV_NO_AOF)) call_flags |= CMD_CALL_PROPAGATE_AOF; diff --git a/src/server.c b/src/server.c index bbc7ba5ff..852fc4ff9 100644 --- a/src/server.c +++ b/src/server.c @@ -3267,11 +3267,15 @@ void call(client *c, int flags) { int multi_emitted = 0; /* Wrap the commands in server.also_propagate array, * but don't wrap it if we are already in MULIT context, - * in case the nested MULIT/EXEC. + * in case the nested MULTI/EXEC. * * And if the array contains only one command, no need to * wrap it, since the single command is atomic. */ - if (server.also_propagate.numops > 1 && !(c->flags & CLIENT_MULTI)) { + if (server.also_propagate.numops > 1 && + !(c->cmd->flags & CMD_MODULE) && + !(c->flags & CLIENT_MULTI) && + !(flags & CMD_CALL_NOWRAP)) + { execCommandPropagateMulti(c); multi_emitted = 1; } diff --git a/src/server.h b/src/server.h index 2c122d259..f4bd4039f 100644 --- a/src/server.h +++ b/src/server.h @@ -395,6 +395,8 @@ typedef long long ustime_t; /* microsecond time type. */ #define CMD_CALL_PROPAGATE_REPL (1<<3) #define CMD_CALL_PROPAGATE (CMD_CALL_PROPAGATE_AOF|CMD_CALL_PROPAGATE_REPL) #define CMD_CALL_FULL (CMD_CALL_SLOWLOG | CMD_CALL_STATS | CMD_CALL_PROPAGATE) +#define CMD_CALL_NOWRAP (1<<4) /* Don't wrap also propagate array into + MULTI/EXEC: the caller will handle it. */ /* Command propagation flags, see propagate() function */ #define PROPAGATE_NONE 0 diff --git a/tests/modules/propagate.c b/tests/modules/propagate.c index f83af1799..2571ee43d 100644 --- a/tests/modules/propagate.c +++ b/tests/modules/propagate.c @@ -89,6 +89,16 @@ int propagateTestCommand(RedisModuleCtx *ctx, RedisModuleString **argv, int argc return REDISMODULE_OK; } +int propagateTest2Command(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) +{ + REDISMODULE_NOT_USED(argv); + REDISMODULE_NOT_USED(argc); + + RedisModule_Replicate(ctx,"INCR","c","counter"); + RedisModule_ReplyWithSimpleString(ctx,"OK"); + return REDISMODULE_OK; +} + int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { REDISMODULE_NOT_USED(argv); REDISMODULE_NOT_USED(argc); @@ -100,5 +110,11 @@ int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) propagateTestCommand, "",1,1,1) == REDISMODULE_ERR) return REDISMODULE_ERR; + + if (RedisModule_CreateCommand(ctx,"propagate-test-2", + propagateTest2Command, + "",1,1,1) == REDISMODULE_ERR) + return REDISMODULE_ERR; + return REDISMODULE_OK; } From 4f6b6b8008b0e3c5ef33876d0599f1189166c5c5 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 31 Mar 2020 12:04:06 +0200 Subject: [PATCH 187/225] Modify the propagate unit test to show more cases. --- tests/modules/propagate.c | 32 ++++++++++++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/tests/modules/propagate.c b/tests/modules/propagate.c index 2571ee43d..13277b19d 100644 --- a/tests/modules/propagate.c +++ b/tests/modules/propagate.c @@ -64,7 +64,8 @@ void *threadMain(void *arg) { RedisModule_SelectDb(ctx,9); /* Tests ran in database number 9. */ for (int i = 0; i < 10; i++) { RedisModule_ThreadSafeContextLock(ctx); - RedisModule_Replicate(ctx,"INCR","c","thread"); + RedisModule_Replicate(ctx,"INCR","c","a-from-thread"); + RedisModule_Replicate(ctx,"INCR","c","b-from-thread"); RedisModule_ThreadSafeContextUnlock(ctx); } RedisModule_FreeThreadSafeContext(ctx); @@ -94,7 +95,29 @@ int propagateTest2Command(RedisModuleCtx *ctx, RedisModuleString **argv, int arg REDISMODULE_NOT_USED(argv); REDISMODULE_NOT_USED(argc); - RedisModule_Replicate(ctx,"INCR","c","counter"); + /* Replicate two commands to test MULTI/EXEC wrapping. */ + RedisModule_Replicate(ctx,"INCR","c","counter-1"); + RedisModule_Replicate(ctx,"INCR","c","counter-2"); + RedisModule_ReplyWithSimpleString(ctx,"OK"); + return REDISMODULE_OK; +} + +int propagateTest3Command(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) +{ + REDISMODULE_NOT_USED(argv); + REDISMODULE_NOT_USED(argc); + RedisModuleCallReply *reply; + + /* This test mixes multiple propagation systems. */ + reply = RedisModule_Call(ctx, "INCR", "c!", "using-call"); + RedisModule_FreeCallReply(reply); + + RedisModule_Replicate(ctx,"INCR","c","counter-1"); + RedisModule_Replicate(ctx,"INCR","c","counter-2"); + + reply = RedisModule_Call(ctx, "INCR", "c!", "after-call"); + RedisModule_FreeCallReply(reply); + RedisModule_ReplyWithSimpleString(ctx,"OK"); return REDISMODULE_OK; } @@ -116,5 +139,10 @@ int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) "",1,1,1) == REDISMODULE_ERR) return REDISMODULE_ERR; + if (RedisModule_CreateCommand(ctx,"propagate-test-3", + propagateTest3Command, + "",1,1,1) == REDISMODULE_ERR) + return REDISMODULE_ERR; + return REDISMODULE_OK; } From 97b80b57e910e809cfc59e1851baebb44d36687a Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 31 Mar 2020 12:09:38 +0200 Subject: [PATCH 188/225] Fix the propagate Tcl test after module changes. --- tests/unit/moduleapi/propagate.tcl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/moduleapi/propagate.tcl b/tests/unit/moduleapi/propagate.tcl index 71307ce33..73f795c71 100644 --- a/tests/unit/moduleapi/propagate.tcl +++ b/tests/unit/moduleapi/propagate.tcl @@ -20,7 +20,7 @@ tags "modules" { wait_for_condition 5000 10 { ([$replica get timer] eq "10") && \ - ([$replica get thread] eq "10") + ([$replica get a-from-thread] eq "10") } else { fail "The two counters don't match the expected value." } From 805c8c94a94e731170cc06c86705108ba3baf3bf Mon Sep 17 00:00:00 2001 From: Guy Benoish Date: Tue, 31 Mar 2020 17:37:05 +0300 Subject: [PATCH 189/225] RENAME can unblock XREADGROUP Other changes: Support stream in serverLogObjectDebugInfo --- src/db.c | 3 ++- src/debug.c | 2 ++ tests/unit/type/stream-cgroups.tcl | 21 +++++++++++++++++++++ 3 files changed, 25 insertions(+), 1 deletion(-) diff --git a/src/db.c b/src/db.c index 04e26c33b..211bb978d 100644 --- a/src/db.c +++ b/src/db.c @@ -182,7 +182,8 @@ void dbAdd(redisDb *db, robj *key, robj *val) { serverAssertWithInfo(NULL,key,retval == DICT_OK); if (val->type == OBJ_LIST || - val->type == OBJ_ZSET) + val->type == OBJ_ZSET || + val->type == OBJ_STREAM) signalKeyAsReady(db, key); if (server.cluster_enabled) slotToKeyAdd(key); } diff --git a/src/debug.c b/src/debug.c index 36af35aec..83e5b6197 100644 --- a/src/debug.c +++ b/src/debug.c @@ -817,6 +817,8 @@ void serverLogObjectDebugInfo(const robj *o) { serverLog(LL_WARNING,"Sorted set size: %d", (int) zsetLength(o)); if (o->encoding == OBJ_ENCODING_SKIPLIST) serverLog(LL_WARNING,"Skiplist level: %d", (int) ((const zset*)o->ptr)->zsl->level); + } else if (o->type == OBJ_STREAM) { + serverLog(LL_WARNING,"Stream size: %d", (int) streamLength(o)); } } diff --git a/tests/unit/type/stream-cgroups.tcl b/tests/unit/type/stream-cgroups.tcl index 6b9a4a9cd..a27e1f582 100644 --- a/tests/unit/type/stream-cgroups.tcl +++ b/tests/unit/type/stream-cgroups.tcl @@ -170,6 +170,27 @@ start_server { assert_error "*NOGROUP*" {$rd read} } + test {RENAME can unblock XREADGROUP with data} { + r del mystream + r XGROUP CREATE mystream mygroup $ MKSTREAM + set rd [redis_deferring_client] + $rd XREADGROUP GROUP mygroup Alice BLOCK 0 STREAMS mystream ">" + r XGROUP CREATE mystream2 mygroup $ MKSTREAM + r XADD mystream2 100 f1 v1 + r RENAME mystream2 mystream + assert_equal "{mystream {{100-0 {f1 v1}}}}" [$rd read] ;# mystream2 had mygroup before RENAME + } + + test {RENAME can unblock XREADGROUP with -NOGROUP} { + r del mystream + r XGROUP CREATE mystream mygroup $ MKSTREAM + set rd [redis_deferring_client] + $rd XREADGROUP GROUP mygroup Alice BLOCK 0 STREAMS mystream ">" + r XADD mystream2 100 f1 v1 + r RENAME mystream2 mystream + assert_error "*NOGROUP*" {$rd read} ;# mystream2 didn't have mygroup before RENAME + } + test {XCLAIM can claim PEL items from another consumer} { # Add 3 items into the stream, and create a consumer group r del mystream From 0f7dfc378ce5ab0524ac28b6142545bc41435386 Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Wed, 4 Dec 2019 17:29:29 +0200 Subject: [PATCH 190/225] AOFRW on an empty stream created with MKSTREAM loads badkly the AOF will be loaded successfully, but the stream will be missing, i.e inconsistencies with the original db. this was because XADD with id of 0-0 would error. add a test to reproduce. --- src/aof.c | 3 ++- tests/unit/type/stream-cgroups.tcl | 13 +++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/src/aof.c b/src/aof.c index 8ab9349f0..6bb239252 100644 --- a/src/aof.c +++ b/src/aof.c @@ -1212,12 +1212,13 @@ int rewriteStreamObject(rio *r, robj *key, robj *o) { /* Use the XADD MAXLEN 0 trick to generate an empty stream if * the key we are serializing is an empty string, which is possible * for the Stream type. */ + id.ms = 0; id.seq = 1; if (rioWriteBulkCount(r,'*',7) == 0) return 0; if (rioWriteBulkString(r,"XADD",4) == 0) return 0; if (rioWriteBulkObject(r,key) == 0) return 0; if (rioWriteBulkString(r,"MAXLEN",6) == 0) return 0; if (rioWriteBulkString(r,"0",1) == 0) return 0; - if (rioWriteBulkStreamID(r,&s->last_id) == 0) return 0; + if (rioWriteBulkStreamID(r,&id) == 0) return 0; if (rioWriteBulkString(r,"x",1) == 0) return 0; if (rioWriteBulkString(r,"y",1) == 0) return 0; } diff --git a/tests/unit/type/stream-cgroups.tcl b/tests/unit/type/stream-cgroups.tcl index 072ed14d6..6b9a4a9cd 100644 --- a/tests/unit/type/stream-cgroups.tcl +++ b/tests/unit/type/stream-cgroups.tcl @@ -311,4 +311,17 @@ start_server { } } } + + start_server {tags {"stream"} overrides {appendonly yes aof-use-rdb-preamble no}} { + test {Empty stream with no lastid can be rewrite into AOF correctly} { + r XGROUP CREATE mystream group-name $ MKSTREAM + assert {[dict get [r xinfo stream mystream] length] == 0} + set grpinfo [r xinfo groups mystream] + r bgrewriteaof + waitForBgrewriteaof r + r debug loadaof + assert {[dict get [r xinfo stream mystream] length] == 0} + assert {[r xinfo groups mystream] == $grpinfo} + } + } } From c3b268a0bcded2d730790a772176483efb7c31fb Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 27 Mar 2020 16:34:45 +0100 Subject: [PATCH 191/225] timeout.c created: move client timeouts code there. --- src/Makefile | 2 +- src/blocked.c | 39 ---------- src/server.c | 128 +-------------------------------- src/server.h | 4 ++ src/timeout.c | 192 ++++++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 198 insertions(+), 167 deletions(-) create mode 100644 src/timeout.c diff --git a/src/Makefile b/src/Makefile index bbfb06440..3f982cc8e 100644 --- a/src/Makefile +++ b/src/Makefile @@ -206,7 +206,7 @@ endif REDIS_SERVER_NAME=redis-server REDIS_SENTINEL_NAME=redis-sentinel -REDIS_SERVER_OBJ=adlist.o quicklist.o ae.o anet.o dict.o server.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o cluster.o crc16.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o memtest.o crc64.o bitops.o sentinel.o notify.o setproctitle.o blocked.o hyperloglog.o latency.o sparkline.o redis-check-rdb.o redis-check-aof.o geo.o lazyfree.o module.o evict.o expire.o geohash.o geohash_helper.o childinfo.o defrag.o siphash.o rax.o t_stream.o listpack.o localtime.o lolwut.o lolwut5.o lolwut6.o acl.o gopher.o tracking.o connection.o tls.o sha256.o +REDIS_SERVER_OBJ=adlist.o quicklist.o ae.o anet.o dict.o server.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o cluster.o crc16.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o memtest.o crc64.o bitops.o sentinel.o notify.o setproctitle.o blocked.o hyperloglog.o latency.o sparkline.o redis-check-rdb.o redis-check-aof.o geo.o lazyfree.o module.o evict.o expire.o geohash.o geohash_helper.o childinfo.o defrag.o siphash.o rax.o t_stream.o listpack.o localtime.o lolwut.o lolwut5.o lolwut6.o acl.o gopher.o tracking.o connection.o tls.o sha256.o timeout.o REDIS_CLI_NAME=redis-cli REDIS_CLI_OBJ=anet.o adlist.o dict.o redis-cli.o zmalloc.o release.o ae.o crc64.o siphash.o crc16.o REDIS_BENCHMARK_NAME=redis-benchmark diff --git a/src/blocked.c b/src/blocked.c index 795985ea1..e3a803ae3 100644 --- a/src/blocked.c +++ b/src/blocked.c @@ -31,9 +31,6 @@ * * API: * - * getTimeoutFromObjectOrReply() is just an utility function to parse a - * timeout argument since blocking operations usually require a timeout. - * * blockClient() set the CLIENT_BLOCKED flag in the client, and set the * specified block type 'btype' filed to one of BLOCKED_* macros. * @@ -67,42 +64,6 @@ int serveClientBlockedOnList(client *receiver, robj *key, robj *dstkey, redisDb *db, robj *value, int where); -/* Get a timeout value from an object and store it into 'timeout'. - * The final timeout is always stored as milliseconds as a time where the - * timeout will expire, however the parsing is performed according to - * the 'unit' that can be seconds or milliseconds. - * - * Note that if the timeout is zero (usually from the point of view of - * commands API this means no timeout) the value stored into 'timeout' - * is zero. */ -int getTimeoutFromObjectOrReply(client *c, robj *object, mstime_t *timeout, int unit) { - long long tval; - long double ftval; - - if (unit == UNIT_SECONDS) { - if (getLongDoubleFromObjectOrReply(c,object,&ftval, - "timeout is not an float or out of range") != C_OK) - return C_ERR; - tval = (long long) (ftval * 1000.0); - } else { - if (getLongLongFromObjectOrReply(c,object,&tval, - "timeout is not an integer or out of range") != C_OK) - return C_ERR; - } - - if (tval < 0) { - addReplyError(c,"timeout is negative"); - return C_ERR; - } - - if (tval > 0) { - tval += mstime(); - } - *timeout = tval; - - return C_OK; -} - /* Block a client for the specific operation type. Once the CLIENT_BLOCKED * flag is set client query buffer is not longer processed, but accumulated, * and will be processed when the client is unblocked. */ diff --git a/src/server.c b/src/server.c index 9a89290ca..bbc7ba5ff 100644 --- a/src/server.c +++ b/src/server.c @@ -1473,132 +1473,6 @@ int allPersistenceDisabled(void) { return server.saveparamslen == 0 && server.aof_state == AOF_OFF; } -/* ========================== Clients timeouts ============================= */ - -/* Check if this blocked client timedout (does nothing if the client is - * not blocked right now). If so send a reply, unblock it, and return 1. - * Otherwise 0 is returned and no operation is performed. */ -int checkBlockedClientTimeout(client *c, mstime_t now) { - if (c->flags & CLIENT_BLOCKED && - c->bpop.timeout != 0 - && c->bpop.timeout < now) - { - /* Handle blocking operation specific timeout. */ - replyToBlockedClientTimedOut(c); - unblockClient(c); - return 1; - } else { - return 0; - } -} - -/* Check for timeouts. Returns non-zero if the client was terminated. - * The function gets the current time in milliseconds as argument since - * it gets called multiple times in a loop, so calling gettimeofday() for - * each iteration would be costly without any actual gain. */ -int clientsCronHandleTimeout(client *c, mstime_t now_ms) { - time_t now = now_ms/1000; - - if (server.maxidletime && - /* This handles the idle clients connection timeout if set. */ - !(c->flags & CLIENT_SLAVE) && /* No timeout for slaves and monitors */ - !(c->flags & CLIENT_MASTER) && /* No timeout for masters */ - !(c->flags & CLIENT_BLOCKED) && /* No timeout for BLPOP */ - !(c->flags & CLIENT_PUBSUB) && /* No timeout for Pub/Sub clients */ - (now - c->lastinteraction > server.maxidletime)) - { - serverLog(LL_VERBOSE,"Closing idle client"); - freeClient(c); - return 1; - } else if (c->flags & CLIENT_BLOCKED) { - /* Cluster: handle unblock & redirect of clients blocked - * into keys no longer served by this server. */ - if (server.cluster_enabled) { - if (clusterRedirectBlockedClientIfNeeded(c)) - unblockClient(c); - } - } - return 0; -} - -/* For blocked clients timeouts we populate a radix tree of 128 bit keys - * composed as such: - * - * [8 byte big endian expire time]+[8 byte client ID] - * - * We don't do any cleanup in the Radix tree: when we run the clients that - * reached the timeout already, if they are no longer existing or no longer - * blocked with such timeout, we just go forward. - * - * Every time a client blocks with a timeout, we add the client in - * the tree. In beforeSleep() we call clientsHandleTimeout() to run - * the tree and unblock the clients. */ - -#define CLIENT_ST_KEYLEN 16 /* 8 bytes mstime + 8 bytes client ID. */ - -/* Given client ID and timeout, write the resulting radix tree key in buf. */ -void encodeTimeoutKey(unsigned char *buf, uint64_t timeout, uint64_t id) { - timeout = htonu64(timeout); - memcpy(buf,&timeout,sizeof(timeout)); - memcpy(buf+8,&id,sizeof(id)); -} - -/* Given a key encoded with encodeTimeoutKey(), resolve the fields and write - * the timeout into *toptr and the client ID into *idptr. */ -void decodeTimeoutKey(unsigned char *buf, uint64_t *toptr, uint64_t *idptr) { - memcpy(toptr,buf,sizeof(*toptr)); - *toptr = ntohu64(*toptr); - memcpy(idptr,buf+8,sizeof(*idptr)); -} - -/* Add the specified client id / timeout as a key in the radix tree we use - * to handle blocked clients timeouts. The client is not added to the list - * if its timeout is zero (block forever). */ -void addClientToTimeoutTable(client *c) { - if (c->bpop.timeout == 0) return; - uint64_t timeout = c->bpop.timeout; - uint64_t id = c->id; - unsigned char buf[CLIENT_ST_KEYLEN]; - encodeTimeoutKey(buf,timeout,id); - if (raxTryInsert(server.clients_timeout_table,buf,sizeof(buf),NULL,NULL)) - c->flags |= CLIENT_IN_TO_TABLE; -} - -/* Remove the client from the table when it is unblocked for reasons - * different than timing out. */ -void removeClientFromTimeoutTable(client *c) { - if (!(c->flags & CLIENT_IN_TO_TABLE)) return; - c->flags &= ~CLIENT_IN_TO_TABLE; - uint64_t timeout = c->bpop.timeout; - uint64_t id = c->id; - unsigned char buf[CLIENT_ST_KEYLEN]; - encodeTimeoutKey(buf,timeout,id); - raxRemove(server.clients_timeout_table,buf,sizeof(buf),NULL); -} - -/* This function is called in beforeSleep() in order to unblock clients - * that are waiting in blocking operations with a timeout set. */ -void clientsHandleTimeout(void) { - if (raxSize(server.clients_timeout_table) == 0) return; - uint64_t now = mstime(); - raxIterator ri; - raxStart(&ri,server.clients_timeout_table); - raxSeek(&ri,"^",NULL,0); - - while(raxNext(&ri)) { - uint64_t id, timeout; - decodeTimeoutKey(ri.key,&timeout,&id); - if (timeout >= now) break; /* All the timeouts are in the future. */ - client *c = lookupClientByID(id); - if (c) { - c->flags &= ~CLIENT_IN_TO_TABLE; - checkBlockedClientTimeout(c,now); - } - raxRemove(server.clients_timeout_table,ri.key,ri.key_len,NULL); - raxSeek(&ri,"^",NULL,0); - } -} - /* ======================= Cron: called every 100 ms ======================== */ /* Add a sample to the operations per second array of samples. */ @@ -2183,7 +2057,7 @@ void beforeSleep(struct aeEventLoop *eventLoop) { UNUSED(eventLoop); /* Handle precise timeouts of blocked clients. */ - clientsHandleTimeout(); + handleBlockedClientsTimeout(); /* We should handle pending reads clients ASAP after event loop. */ handleClientsWithPendingReadsUsingThreads(); diff --git a/src/server.h b/src/server.h index 03a153c7b..2c122d259 100644 --- a/src/server.h +++ b/src/server.h @@ -2139,8 +2139,12 @@ void disconnectAllBlockedClients(void); void handleClientsBlockedOnKeys(void); void signalKeyAsReady(redisDb *db, robj *key); void blockForKeys(client *c, int btype, robj **keys, int numkeys, mstime_t timeout, robj *target, streamID *ids); + +/* timeout.c -- Blocked clients timeout and connections timeout. */ void addClientToTimeoutTable(client *c); void removeClientFromTimeoutTable(client *c); +void handleBlockedClientsTimeout(void); +int clientsCronHandleTimeout(client *c, mstime_t now_ms); /* expire.c -- Handling of expired keys */ void activeExpireCycle(int type); diff --git a/src/timeout.c b/src/timeout.c new file mode 100644 index 000000000..ea2032e2a --- /dev/null +++ b/src/timeout.c @@ -0,0 +1,192 @@ +/* Copyright (c) 2009-2020, Salvatore Sanfilippo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include "server.h" +#include "cluster.h" + +/* ========================== Clients timeouts ============================= */ + +/* Check if this blocked client timedout (does nothing if the client is + * not blocked right now). If so send a reply, unblock it, and return 1. + * Otherwise 0 is returned and no operation is performed. */ +int checkBlockedClientTimeout(client *c, mstime_t now) { + if (c->flags & CLIENT_BLOCKED && + c->bpop.timeout != 0 + && c->bpop.timeout < now) + { + /* Handle blocking operation specific timeout. */ + replyToBlockedClientTimedOut(c); + unblockClient(c); + return 1; + } else { + return 0; + } +} + +/* Check for timeouts. Returns non-zero if the client was terminated. + * The function gets the current time in milliseconds as argument since + * it gets called multiple times in a loop, so calling gettimeofday() for + * each iteration would be costly without any actual gain. */ +int clientsCronHandleTimeout(client *c, mstime_t now_ms) { + time_t now = now_ms/1000; + + if (server.maxidletime && + /* This handles the idle clients connection timeout if set. */ + !(c->flags & CLIENT_SLAVE) && /* No timeout for slaves and monitors */ + !(c->flags & CLIENT_MASTER) && /* No timeout for masters */ + !(c->flags & CLIENT_BLOCKED) && /* No timeout for BLPOP */ + !(c->flags & CLIENT_PUBSUB) && /* No timeout for Pub/Sub clients */ + (now - c->lastinteraction > server.maxidletime)) + { + serverLog(LL_VERBOSE,"Closing idle client"); + freeClient(c); + return 1; + } else if (c->flags & CLIENT_BLOCKED) { + /* Cluster: handle unblock & redirect of clients blocked + * into keys no longer served by this server. */ + if (server.cluster_enabled) { + if (clusterRedirectBlockedClientIfNeeded(c)) + unblockClient(c); + } + } + return 0; +} + +/* For blocked clients timeouts we populate a radix tree of 128 bit keys + * composed as such: + * + * [8 byte big endian expire time]+[8 byte client ID] + * + * We don't do any cleanup in the Radix tree: when we run the clients that + * reached the timeout already, if they are no longer existing or no longer + * blocked with such timeout, we just go forward. + * + * Every time a client blocks with a timeout, we add the client in + * the tree. In beforeSleep() we call handleBlockedClientsTimeout() to run + * the tree and unblock the clients. */ + +#define CLIENT_ST_KEYLEN 16 /* 8 bytes mstime + 8 bytes client ID. */ + +/* Given client ID and timeout, write the resulting radix tree key in buf. */ +void encodeTimeoutKey(unsigned char *buf, uint64_t timeout, uint64_t id) { + timeout = htonu64(timeout); + memcpy(buf,&timeout,sizeof(timeout)); + memcpy(buf+8,&id,sizeof(id)); +} + +/* Given a key encoded with encodeTimeoutKey(), resolve the fields and write + * the timeout into *toptr and the client ID into *idptr. */ +void decodeTimeoutKey(unsigned char *buf, uint64_t *toptr, uint64_t *idptr) { + memcpy(toptr,buf,sizeof(*toptr)); + *toptr = ntohu64(*toptr); + memcpy(idptr,buf+8,sizeof(*idptr)); +} + +/* Add the specified client id / timeout as a key in the radix tree we use + * to handle blocked clients timeouts. The client is not added to the list + * if its timeout is zero (block forever). */ +void addClientToTimeoutTable(client *c) { + if (c->bpop.timeout == 0) return; + uint64_t timeout = c->bpop.timeout; + uint64_t id = c->id; + unsigned char buf[CLIENT_ST_KEYLEN]; + encodeTimeoutKey(buf,timeout,id); + if (raxTryInsert(server.clients_timeout_table,buf,sizeof(buf),NULL,NULL)) + c->flags |= CLIENT_IN_TO_TABLE; +} + +/* Remove the client from the table when it is unblocked for reasons + * different than timing out. */ +void removeClientFromTimeoutTable(client *c) { + if (!(c->flags & CLIENT_IN_TO_TABLE)) return; + c->flags &= ~CLIENT_IN_TO_TABLE; + uint64_t timeout = c->bpop.timeout; + uint64_t id = c->id; + unsigned char buf[CLIENT_ST_KEYLEN]; + encodeTimeoutKey(buf,timeout,id); + raxRemove(server.clients_timeout_table,buf,sizeof(buf),NULL); +} + +/* This function is called in beforeSleep() in order to unblock clients + * that are waiting in blocking operations with a timeout set. */ +void handleBlockedClientsTimeout(void) { + if (raxSize(server.clients_timeout_table) == 0) return; + uint64_t now = mstime(); + raxIterator ri; + raxStart(&ri,server.clients_timeout_table); + raxSeek(&ri,"^",NULL,0); + + while(raxNext(&ri)) { + uint64_t id, timeout; + decodeTimeoutKey(ri.key,&timeout,&id); + if (timeout >= now) break; /* All the timeouts are in the future. */ + client *c = lookupClientByID(id); + if (c) { + c->flags &= ~CLIENT_IN_TO_TABLE; + checkBlockedClientTimeout(c,now); + } + raxRemove(server.clients_timeout_table,ri.key,ri.key_len,NULL); + raxSeek(&ri,"^",NULL,0); + } +} + +/* Get a timeout value from an object and store it into 'timeout'. + * The final timeout is always stored as milliseconds as a time where the + * timeout will expire, however the parsing is performed according to + * the 'unit' that can be seconds or milliseconds. + * + * Note that if the timeout is zero (usually from the point of view of + * commands API this means no timeout) the value stored into 'timeout' + * is zero. */ +int getTimeoutFromObjectOrReply(client *c, robj *object, mstime_t *timeout, int unit) { + long long tval; + long double ftval; + + if (unit == UNIT_SECONDS) { + if (getLongDoubleFromObjectOrReply(c,object,&ftval, + "timeout is not an float or out of range") != C_OK) + return C_ERR; + tval = (long long) (ftval * 1000.0); + } else { + if (getLongLongFromObjectOrReply(c,object,&tval, + "timeout is not an integer or out of range") != C_OK) + return C_ERR; + } + + if (tval < 0) { + addReplyError(c,"timeout is negative"); + return C_ERR; + } + + if (tval > 0) { + tval += mstime(); + } + *timeout = tval; + + return C_OK; +} From a509400d58eb5d0f19bb0908b45d07f1305e36ba Mon Sep 17 00:00:00 2001 From: Guy Benoish Date: Sun, 29 Mar 2020 13:08:21 +0300 Subject: [PATCH 192/225] Modules: Test MULTI/EXEC replication of RM_Replicate Makse sure call() doesn't wrap replicated commands with a redundant MULTI/EXEC Other, unrelated changes: 1. Formatting compiler warning in INFO CLIENTS 2. Use CLIENT_ID_AOF instead of UINT64_MAX --- src/aof.c | 2 +- src/networking.c | 9 +++++--- src/redismodule.h | 2 +- src/server.c | 11 ++++++---- src/server.h | 1 + tests/unit/moduleapi/propagate.tcl | 33 ++++++++++++++++++++++++++++++ 6 files changed, 49 insertions(+), 9 deletions(-) diff --git a/src/aof.c b/src/aof.c index 6bb239252..301a40848 100644 --- a/src/aof.c +++ b/src/aof.c @@ -831,7 +831,7 @@ int loadAppendOnlyFile(char *filename) { if (cmd == server.multiCommand) valid_before_multi = valid_up_to; /* Run the command in the context of a fake client */ - fakeClient->cmd = cmd; + fakeClient->cmd = fakeClient->lastcmd = cmd; if (fakeClient->flags & CLIENT_MULTI && fakeClient->cmd->proc != execCommand) { diff --git a/src/networking.c b/src/networking.c index a550e4040..fcaa164a9 100644 --- a/src/networking.c +++ b/src/networking.c @@ -373,13 +373,16 @@ void addReplyErrorLength(client *c, const char *s, size_t len) { * will produce an error. However it is useful to log such events since * they are rare and may hint at errors in a script or a bug in Redis. */ int ctype = getClientType(c); - if (ctype == CLIENT_TYPE_MASTER || ctype == CLIENT_TYPE_SLAVE) { - char* to = ctype == CLIENT_TYPE_MASTER? "master": "replica"; - char* from = ctype == CLIENT_TYPE_MASTER? "replica": "master"; + if (ctype == CLIENT_TYPE_MASTER || ctype == CLIENT_TYPE_SLAVE || c->id == CLIENT_ID_AOF) { + char* to = c->id == CLIENT_ID_AOF ? "AOF-client" : + ctype == CLIENT_TYPE_MASTER ? "master" : "replica"; + char* from = c->id == CLIENT_ID_AOF ? "server" : + ctype == CLIENT_TYPE_MASTER ? "replica" : "master"; char *cmdname = c->lastcmd ? c->lastcmd->name : ""; serverLog(LL_WARNING,"== CRITICAL == This %s is sending an error " "to its %s: '%s' after processing the command " "'%s'", from, to, s, cmdname); + server.stat_unexpected_error_replies++; } } diff --git a/src/redismodule.h b/src/redismodule.h index a43443f13..d26c41456 100644 --- a/src/redismodule.h +++ b/src/redismodule.h @@ -673,7 +673,7 @@ int REDISMODULE_API_FUNC(RedisModule_AuthenticateClientWithUser)(RedisModuleCtx void REDISMODULE_API_FUNC(RedisModule_DeauthenticateAndCloseClient)(RedisModuleCtx *ctx, uint64_t client_id); #endif -#define RedisModule_IsAOFClient(id) ((id) == UINT64_MAX) +#define RedisModule_IsAOFClient(id) ((id) == CLIENT_ID_AOF) /* This is included inline inside each Redis module. */ static int RedisModule_Init(RedisModuleCtx *ctx, const char *name, int ver, int apiver) __attribute__((unused)); diff --git a/src/server.c b/src/server.c index 852fc4ff9..6f1913e4d 100644 --- a/src/server.c +++ b/src/server.c @@ -2661,6 +2661,7 @@ void resetServerStats(void) { } server.stat_net_input_bytes = 0; server.stat_net_output_bytes = 0; + server.stat_unexpected_error_replies = 0; server.aof_delayed_fsync = 0; } @@ -3266,7 +3267,7 @@ void call(client *c, int flags) { if (flags & CMD_CALL_PROPAGATE) { int multi_emitted = 0; /* Wrap the commands in server.also_propagate array, - * but don't wrap it if we are already in MULIT context, + * but don't wrap it if we are already in MULTI context, * in case the nested MULTI/EXEC. * * And if the array contains only one command, no need to @@ -3975,7 +3976,7 @@ sds genRedisInfoString(const char *section) { "client_recent_max_output_buffer:%zu\r\n" "blocked_clients:%d\r\n" "tracking_clients:%d\r\n" - "clients_in_timeout_table:%lld\r\n", + "clients_in_timeout_table:%ld\r\n", listLength(server.clients)-listLength(server.slaves), maxin, maxout, server.blocked_clients, @@ -4230,7 +4231,8 @@ sds genRedisInfoString(const char *section) { "active_defrag_key_hits:%lld\r\n" "active_defrag_key_misses:%lld\r\n" "tracking_total_keys:%lld\r\n" - "tracking_total_items:%lld\r\n", + "tracking_total_items:%lld\r\n" + "unexpected_error_replies:%lld\r\n", server.stat_numconnections, server.stat_numcommands, getInstantaneousMetric(STATS_METRIC_COMMAND), @@ -4259,7 +4261,8 @@ sds genRedisInfoString(const char *section) { server.stat_active_defrag_key_hits, server.stat_active_defrag_key_misses, (unsigned long long) trackingGetTotalKeys(), - (unsigned long long) trackingGetTotalItems()); + (unsigned long long) trackingGetTotalItems(), + server.stat_unexpected_error_replies); } /* Replication */ diff --git a/src/server.h b/src/server.h index f4bd4039f..c4db4278e 100644 --- a/src/server.h +++ b/src/server.h @@ -1129,6 +1129,7 @@ struct redisServer { size_t stat_rdb_cow_bytes; /* Copy on write bytes during RDB saving. */ size_t stat_aof_cow_bytes; /* Copy on write bytes during AOF rewrite. */ size_t stat_module_cow_bytes; /* Copy on write bytes during module fork. */ + long long stat_unexpected_error_replies; /* Number of unexpected (aof-loading, replica to master, etc.) error replies */ /* The following two are used to track instantaneous metrics, like * number of operations per second, network traffic. */ struct { diff --git a/tests/unit/moduleapi/propagate.tcl b/tests/unit/moduleapi/propagate.tcl index 73f795c71..aa0f55e5e 100644 --- a/tests/unit/moduleapi/propagate.tcl +++ b/tests/unit/moduleapi/propagate.tcl @@ -24,7 +24,40 @@ tags "modules" { } else { fail "The two counters don't match the expected value." } + + $master propagate-test-2 + $master propagate-test-3 + $master multi + $master propagate-test-2 + $master propagate-test-3 + $master exec + wait_for_ofs_sync $master $replica + + assert_equal [s -1 unexpected_error_replies] 0 } } } } + +tags "modules aof" { + test {Modules RM_Replicate replicates MULTI/EXEC correctly} { + start_server [list overrides [list loadmodule "$testmodule"]] { + # Enable the AOF + r config set appendonly yes + r config set auto-aof-rewrite-percentage 0 ; # Disable auto-rewrite. + waitForBgrewriteaof r + + r propagate-test-2 + r propagate-test-3 + r multi + r propagate-test-2 + r propagate-test-3 + r exec + + # Load the AOF + r debug loadaof + + assert_equal [s 0 unexpected_error_replies] 0 + } + } +} From 9f347fabba7c2a3a09cba8145068563e3da5f28a Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 31 Mar 2020 17:10:09 +0200 Subject: [PATCH 193/225] Minor changes to #7037. --- src/networking.c | 17 +++++++++++++---- src/server.c | 2 +- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/src/networking.c b/src/networking.c index fcaa164a9..3c754c376 100644 --- a/src/networking.c +++ b/src/networking.c @@ -374,10 +374,19 @@ void addReplyErrorLength(client *c, const char *s, size_t len) { * they are rare and may hint at errors in a script or a bug in Redis. */ int ctype = getClientType(c); if (ctype == CLIENT_TYPE_MASTER || ctype == CLIENT_TYPE_SLAVE || c->id == CLIENT_ID_AOF) { - char* to = c->id == CLIENT_ID_AOF ? "AOF-client" : - ctype == CLIENT_TYPE_MASTER ? "master" : "replica"; - char* from = c->id == CLIENT_ID_AOF ? "server" : - ctype == CLIENT_TYPE_MASTER ? "replica" : "master"; + char *to, *from; + + if (c->id == CLIENT_ID_AOF) { + to = "AOF-loading-client"; + from = "server"; + } else if (ctype == CLIENT_TYPE_MASTER) { + to = "master"; + from = "replica"; + } else { + to = "replica"; + from = "master"; + } + char *cmdname = c->lastcmd ? c->lastcmd->name : ""; serverLog(LL_WARNING,"== CRITICAL == This %s is sending an error " "to its %s: '%s' after processing the command " diff --git a/src/server.c b/src/server.c index 6f1913e4d..4fdbf30ec 100644 --- a/src/server.c +++ b/src/server.c @@ -3976,7 +3976,7 @@ sds genRedisInfoString(const char *section) { "client_recent_max_output_buffer:%zu\r\n" "blocked_clients:%d\r\n" "tracking_clients:%d\r\n" - "clients_in_timeout_table:%ld\r\n", + "clients_in_timeout_table:%llu\r\n", listLength(server.clients)-listLength(server.slaves), maxin, maxout, server.blocked_clients, From ef1b1f01a84e969ea368e7fdbaf0d10615743269 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 31 Mar 2020 17:41:23 +0200 Subject: [PATCH 194/225] cast raxSize() to avoid warning with format spec. --- src/server.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/server.c b/src/server.c index 4fdbf30ec..c89e9c075 100644 --- a/src/server.c +++ b/src/server.c @@ -3981,7 +3981,7 @@ sds genRedisInfoString(const char *section) { maxin, maxout, server.blocked_clients, server.tracking_clients, - raxSize(server.clients_timeout_table)); + (unsigned long long) raxSize(server.clients_timeout_table)); } /* Memory */ From 957e917a84ac9979f18145a4d0b53386f5ce4fd9 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 31 Mar 2020 17:56:04 +0200 Subject: [PATCH 195/225] Redis 6.0-RC3. --- 00-RELEASENOTES | 317 ++++++++++++++++++++++++++++++++++++++++++++++++ src/version.h | 2 +- 2 files changed, 318 insertions(+), 1 deletion(-) diff --git a/00-RELEASENOTES b/00-RELEASENOTES index 9847d37dd..290158efb 100644 --- a/00-RELEASENOTES +++ b/00-RELEASENOTES @@ -11,6 +11,323 @@ CRITICAL: There is a critical bug affecting MOST USERS. Upgrade ASAP. SECURITY: There are security fixes in the release. -------------------------------------------------------------------------------- +================================================================================ +Redis 6.0-rc3 Released Tue Mar 31 17:42:39 CEST 2020 +================================================================================ + +Upgrade urgency CRITICAL: A connection management bug introduced with the + SSL implementation can crash Redis easily. + +Dear users, this is a list of the major changes in this release, please check +the list of commits for detail: + +* Fix crash due to refactoring for SSL, for the connection code. +* Precise timeouts for blocking commands. Now the timeouts have HZ + resolution regardless of the number of connected clinets. New timeouts + are stored in a radix tree and sorted by expire time. +* Fix rare crash when resizing the event loop because of CONFIG maxclients. +* Fix systemd readiness after successful partial resync. +* Redis-cli ask password mode to be prompted at startup (for additional safety). +* Keyspace notifications added to MIGRATE / RESTORE. +* Threaded I/O bugs fixed. +* Implement new ACL style AUTH in Sentinel. +* Make 'requirepass' more backward compatible with Redis <= 5. +* ACL: Handle default user as disabled if it's off regardless of "nopass". +* Fix a potential inconsistency when upgrading an instance in Redis Cluster + and restarting it. The instance will act as a replica but will actually be + set as a master immediately. However the choice of what to do with already + expired keys, on loading, was made from the POV of replicas. +* Abort transactions after -READONLY error. +* Many different fixes to module APIs. +* BITFIELD_RO added to call the command on read only replicas. +* PSYNC2: meaningful offset implementation. Allow the disconnected master + that is still sending PINGs to replicas, to be able to successfully + PSYNC incrementally to new slaves, discarding the last part of the + replication backlog consisting only of PINGs. +* Fix pipelined MULTI/EXEC during Lua scripts are in BUSY state. +* Re-fix propagation API in modules, broken again after other changes. + +antirez in commit ef1b1f01: + cast raxSize() to avoid warning with format spec. + 1 file changed, 1 insertion(+), 1 deletion(-) + +antirez in commit 9f347fab: + Minor changes to #7037. + 2 files changed, 14 insertions(+), 5 deletions(-) + +Guy Benoish in commit a509400d: + Modules: Test MULTI/EXEC replication of RM_Replicate + 6 files changed, 49 insertions(+), 9 deletions(-) + +Guy Benoish in commit 805c8c94: + RENAME can unblock XREADGROUP + 3 files changed, 25 insertions(+), 1 deletion(-) + +antirez in commit 97b80b57: + Fix the propagate Tcl test after module changes. + 1 file changed, 1 insertion(+), 1 deletion(-) + +antirez in commit 4f6b6b80: + Modify the propagate unit test to show more cases. + 1 file changed, 30 insertions(+), 2 deletions(-) + +antirez in commit 616b1cb7: + Fix module commands propagation double MULTI bug. + 4 files changed, 25 insertions(+), 8 deletions(-) + +antirez in commit 08fdef4b: + Fix RM_Call() stale comment due to cut&paste. + 1 file changed, 1 insertion(+), 3 deletions(-) + +OMG-By in commit 26b79ca1: + fix: dict.c->dictResize()->minimal type + 1 file changed, 1 insertion(+), 1 deletion(-) + +zhaozhao.zz in commit fa418637: + PSYNC2: reset backlog_idx and master_repl_offset correctly + 1 file changed, 10 insertions(+), 5 deletions(-) + +antirez in commit bbbc80ac: + Precise timeouts: reference client pointer directly. + 1 file changed, 13 insertions(+), 16 deletions(-) + +antirez in commit c3b268a0: + timeout.c created: move client timeouts code there. + 5 files changed, 198 insertions(+), 167 deletions(-) + +Oran Agra in commit 0f7dfc37: + AOFRW on an empty stream created with MKSTREAM loads badkly + 2 files changed, 15 insertions(+), 1 deletion(-) + +antirez in commit 67643ead: + Precise timeouts: cleaup the table on unblock. + 3 files changed, 21 insertions(+), 2 deletions(-) + +antirez in commit ad94066e: + Precise timeouts: fix comments after functional change. + 2 files changed, 6 insertions(+), 6 deletions(-) + +antirez in commit a443ec2e: + Precise timeouts: use only radix tree for timeouts. + 3 files changed, 15 insertions(+), 38 deletions(-) + +antirez in commit 6862fd70: + Precise timeouts: fast exit for clientsHandleShortTimeout(). + 1 file changed, 1 insertion(+) + +antirez in commit 30f1df8c: + Precise timeouts: fix bugs in initial implementation. + 2 files changed, 5 insertions(+), 1 deletion(-) + +antirez in commit 7add0f24: + Precise timeouts: working initial implementation. + 3 files changed, 110 insertions(+), 28 deletions(-) + +antirez in commit 9d6d1779: + Precise timeouts: refactor unblocking on timeout. + 2 files changed, 33 insertions(+), 13 deletions(-) + +antirez in commit 316a8f15: + PSYNC2: fix backlog_idx when adjusting for meaningful offset + 1 file changed, 3 insertions(+) + +伯成 in commit 11db53f8: + Boost up performance for redis PUB-SUB patterns matching + 3 files changed, 43 insertions(+), 11 deletions(-) + +antirez in commit e257f121: + PSYNC2: meaningful offset test. + 2 files changed, 62 insertions(+) + +antirez in commit 5f72f696: + PSYNC2: meaningful offset implemented. + 3 files changed, 40 insertions(+), 1 deletion(-) + +antirez in commit 8caa2714: + Explain why we allow transactions in -BUSY state. + 1 file changed, 9 insertions(+), 2 deletions(-) + +Oran Agra in commit e43cd831: + MULTI/EXEC during LUA script timeout are messed up + 2 files changed, 73 insertions(+) + +antirez in commit 34b89832: + Improve comments of replicationCacheMasterUsingMyself(). + 1 file changed, 6 insertions(+), 1 deletion(-) + +antirez in commit 70a98a43: + Fix BITFIELD_RO test. + 2 files changed, 5 insertions(+), 5 deletions(-) + +antirez in commit 8783304a: + Abort transactions after -READONLY error. Fix #7014. + 1 file changed, 1 insertion(+) + +antirez in commit ec9cf002: + Minor changes to BITFIELD_RO PR #6951. + 1 file changed, 9 insertions(+), 6 deletions(-) + +bodong.ybd in commit b3e4abf0: + Added BITFIELD_RO variants for read-only operations. + 4 files changed, 54 insertions(+), 1 deletion(-) + +antirez in commit 50f8f950: + Modules: updated function doc after #7003. + 1 file changed, 6 insertions(+), 1 deletion(-) + +Guy Benoish in commit f2f3dc5e: + Allow RM_GetContextFlags to work with ctx==NULL + 1 file changed, 16 insertions(+), 14 deletions(-) + +hwware in commit eb808879: + fix potentical memory leak in redis-cli + 1 file changed, 2 insertions(+) + +Yossi Gottlieb in commit cdcab0e8: + Fix crashes related to failed/rejected accepts. + 1 file changed, 6 insertions(+), 5 deletions(-) + +Yossi Gottlieb in commit 50dcd9f9: + Cluster: fix misleading accept errors. + 1 file changed, 4 insertions(+), 3 deletions(-) + +Yossi Gottlieb in commit 87dbd8f5: + Conns: Fix connClose() / connAccept() behavior. + 3 files changed, 48 insertions(+), 32 deletions(-) + +hwware in commit 81e8686c: + remove redundant Semicolon + 1 file changed, 1 insertion(+), 1 deletion(-) + +hwware in commit c7524a7e: + clean CLIENT_TRACKING_CACHING flag when disabled caching + 1 file changed, 1 insertion(+), 1 deletion(-) + +hwware in commit 2dd1ca6a: + add missing commands in cluster help + 1 file changed, 2 insertions(+), 1 deletion(-) + +artix in commit 95324b81: + Support Redis Cluster Proxy PROXY INFO command + 1 file changed, 5 insertions(+), 1 deletion(-) + +박승현 in commit 04c53fa1: + Update redis.conf + 1 file changed, 1 insertion(+), 1 deletion(-) + +WuYunlong in commit 0578157d: + Fix master replica inconsistency for upgrading scenario. + 3 files changed, 9 insertions(+), 2 deletions(-) + +WuYunlong in commit 299f1d02: + Add 14-consistency-check.tcl to prove there is a data consistency issue. + 1 file changed, 87 insertions(+) + +antirez in commit 61b98f32: + Regression test for #7011. + 1 file changed, 7 insertions(+) + +antirez in commit 34ea2f4e: + ACL: default user off should not allow automatic authentication. + 2 files changed, 3 insertions(+), 2 deletions(-) + +antirez in commit cbbf9b39: + Sentinel: document auth-user directive. + 1 file changed, 12 insertions(+) + +antirez in commit 9c2e42dd: + ACL: Make Redis 6 more backward compatible with requirepass. + 4 files changed, 17 insertions(+), 15 deletions(-) + +antirez in commit d387f67d: + Sentinel: implement auth-user directive for ACLs. + 1 file changed, 38 insertions(+), 7 deletions(-) + +zhaozhao.zz in commit 7c078416: + Threaded IO: bugfix client kill may crash redis + 1 file changed, 11 insertions(+), 5 deletions(-) + +zhaozhao.zz in commit 9cc7038e: + Threaded IO: handle pending reads clients ASAP after event loop + 1 file changed, 3 insertions(+), 1 deletion(-) + +antirez in commit da8c7c49: + Example sentinel conf: document requirepass. + 1 file changed, 8 insertions(+) + +antirez in commit bdb338cf: + Aesthetic changes in PR #6989. + 1 file changed, 9 insertions(+), 5 deletions(-) + +zhaozhao.zz in commit b3e03054: + Threaded IO: bugfix #6988 process events while blocked + 1 file changed, 5 insertions(+) + +antirez in commit e628f944: + Restore newline at the end of redis-cli.c + 1 file changed, 2 insertions(+), 1 deletion(-) + +chendianqiang in commit 5d4c4df3: + use correct list for moduleUnregisterUsedAPI + 1 file changed, 1 insertion(+), 1 deletion(-) + +guodongxiaren in commit da14982d: + string literal should be const char* + 1 file changed, 1 insertion(+), 1 deletion(-) + +Itamar Haber in commit dc8885a1: + Adds keyspace notifications to migrate and restore + 1 file changed, 3 insertions(+), 1 deletion(-) + +bodong.ybd in commit bfb18e55: + Remove duplicate obj files in Makefile + 1 file changed, 2 insertions(+), 2 deletions(-) + +bodong.ybd in commit 76d57161: + Fix bug of tcl test using external server + 2 files changed, 8 insertions(+), 2 deletions(-) + +fengpf in commit 0e5820d8: + fix comments in latency.c + 2 files changed, 2 insertions(+), 1 deletion(-) + +antirez in commit 916dd79f: + Update linenoise. + 1 file changed, 2 insertions(+), 1 deletion(-) + +lifubang in commit c0c67c9b: + add askpass mode + 1 file changed, 19 insertions(+), 1 deletion(-) + +lifubang in commit e1c29434: + update linenoise to https://github.com/antirez/linenoise/tree/fc9667a81d43911a6690fb1e68c16e6e3bb8df05 + 4 files changed, 59 insertions(+), 4 deletions(-) + +Jamie Scott in commit e5a063bc: + Remove default guidance in Redis.conf + 1 file changed, 1 insertion(+), 2 deletions(-) + +Jamie Scott in commit d28cbaf7: + Update Redis.conf to improve TLS usability + 1 file changed, 2 insertions(+), 1 deletion(-) + +Johannes Truschnigg in commit 23d5e8b8: + Signal systemd readiness atfer Partial Resync + 1 file changed, 4 insertions(+) + +Oran Agra in commit 61738154: + fix for flaky psync2 test + 1 file changed, 21 insertions(+) + +antirez in commit 70e0e499: + ae.c: fix crash when resizing the event loop. + 1 file changed, 6 insertions(+), 2 deletions(-) + +antirez in commit b3e4aa67: + Fix release notes spelling mistake. + 1 file changed, 1 insertion(+), 1 deletion(-) + + ================================================================================ Redis 6.0 RC2 Released Thu Mar 05 15:40:53 CET 2020 ================================================================================ diff --git a/src/version.h b/src/version.h index 8db18a071..45465341c 100644 --- a/src/version.h +++ b/src/version.h @@ -1 +1 @@ -#define REDIS_VERSION "5.9.102" +#define REDIS_VERSION "5.9.103" From eeca58777defd34de29e6da8bcdf8865226ab501 Mon Sep 17 00:00:00 2001 From: John Sully Date: Sat, 4 Apr 2020 21:52:27 -0400 Subject: [PATCH 196/225] Fix subkey expires not replicating correctly, and AOF issues Former-commit-id: bd183cdee13081a02efef5df75edf2292b872a16 --- src/aof.cpp | 89 +++++++++++--- src/cron.cpp | 12 ++ src/db.cpp | 43 +++++++ src/debug.cpp | 5 + src/expire.cpp | 30 +++-- src/replication.cpp | 24 +++- src/server.cpp | 9 ++ src/server.h | 8 +- tests/integration/aof.tcl | 22 ++++ tests/integration/replication-4.tcl | 88 ++++++++++++++ tests/integration/replication-active.tcl | 145 ++++++++++++++++++----- 11 files changed, 417 insertions(+), 58 deletions(-) diff --git a/src/aof.cpp b/src/aof.cpp index de8a8260e..4a7eb26ed 100644 --- a/src/aof.cpp +++ b/src/aof.cpp @@ -597,21 +597,59 @@ sds catAppendOnlyExpireAtCommand(sds buf, struct redisCommand *cmd, robj *key, r return buf; } -void feedAppendOnlyFile(struct redisCommand *cmd, int dictid, robj **argv, int argc) { - sds buf = sdsempty(); - robj *tmpargv[3]; - - /* The DB this command was targeting is not the same as the last command - * we appended. To issue a SELECT command is needed. */ - if (dictid != g_pserver->aof_selected_db) { - char seldb[64]; - - snprintf(seldb,sizeof(seldb),"%d",dictid); - buf = sdscatprintf(buf,"*2\r\n$6\r\nSELECT\r\n$%lu\r\n%s\r\n", - (unsigned long)strlen(seldb),seldb); - g_pserver->aof_selected_db = dictid; +sds catAppendOnlyExpireMemberAtCommand(sds buf, struct redisCommand *cmd, robj **argv, const size_t argc) { + long long when = 0; + int unit = UNIT_SECONDS; + bool fAbsolute = false; + + if (cmd->proc == expireMemberCommand) { + if (getLongLongFromObject(argv[3], &when) != C_OK) + serverPanic("propogating invalid EXPIREMEMBER command"); + + if (argc == 5) { + unit = parseUnitString(szFromObj(argv[4])); + } + } else if (cmd->proc == expireMemberAtCommand) { + if (getLongLongFromObject(argv[3], &when) != C_OK) + serverPanic("propogating invalid EXPIREMEMBERAT command"); + fAbsolute = true; + } else if (cmd->proc == pexpireMemberAtCommand) { + if (getLongLongFromObject(argv[3], &when) != C_OK) + serverPanic("propogating invalid PEXPIREMEMBERAT command"); + fAbsolute = true; + unit = UNIT_MILLISECONDS; + } else { + serverPanic("Unknown expiremember command"); } + switch (unit) + { + case UNIT_SECONDS: + when *= 1000; + break; + + case UNIT_MILLISECONDS: + break; + } + + if (!fAbsolute) + when += mstime(); + + robj *argvNew[4]; + argvNew[0] = createStringObject("PEXPIREMEMBERAT",15); + argvNew[1] = argv[1]; + argvNew[2] = argv[2]; + argvNew[3] = createStringObjectFromLongLong(when); + buf = catAppendOnlyGenericCommand(buf, 4, argvNew); + decrRefCount(argvNew[0]); + decrRefCount(argvNew[3]); + return buf; +} + +sds catCommandForAofAndActiveReplication(sds buf, struct redisCommand *cmd, robj **argv, int argc) +{ + robj *tmpargv[3]; + if (cmd->proc == expireCommand || cmd->proc == pexpireCommand || cmd->proc == expireatCommand) { /* Translate EXPIRE/PEXPIRE/EXPIREAT into PEXPIREAT */ @@ -640,6 +678,10 @@ void feedAppendOnlyFile(struct redisCommand *cmd, int dictid, robj **argv, int a if (pxarg) buf = catAppendOnlyExpireAtCommand(buf,cserver.pexpireCommand,argv[1], pxarg); + } else if (cmd->proc == expireMemberCommand || cmd->proc == expireMemberAtCommand || + cmd->proc == pexpireMemberAtCommand) { + /* Translate subkey expire commands to PEXPIREMEMBERAT */ + buf = catAppendOnlyExpireMemberAtCommand(buf, cmd, argv, argc); } else { /* All the other commands don't need translation or need the * same translation already operated in the command vector @@ -647,6 +689,25 @@ void feedAppendOnlyFile(struct redisCommand *cmd, int dictid, robj **argv, int a buf = catAppendOnlyGenericCommand(buf,argc,argv); } + return buf; +} + +void feedAppendOnlyFile(struct redisCommand *cmd, int dictid, robj **argv, int argc) { + sds buf = sdsempty(); + + /* The DB this command was targeting is not the same as the last command + * we appended. To issue a SELECT command is needed. */ + if (dictid != g_pserver->aof_selected_db) { + char seldb[64]; + + snprintf(seldb,sizeof(seldb),"%d",dictid); + buf = sdscatprintf(buf,"*2\r\n$6\r\nSELECT\r\n$%lu\r\n%s\r\n", + (unsigned long)strlen(seldb),seldb); + g_pserver->aof_selected_db = dictid; + } + + buf = catCommandForAofAndActiveReplication(buf, cmd, argv, argc); + /* Append to the AOF buffer. This will be flushed on disk just before * of re-entering the event loop, so before the client will get a * positive reply about the operation performed. */ @@ -1378,7 +1439,7 @@ int rewriteAppendOnlyFileRio(rio *aof) { } else { - char cmd[]="*4\r\n$12\r\nEXPIREMEMBER\r\n"; + char cmd[]="*4\r\n$12\r\nPEXPIREMEMBERAT\r\n"; if (rioWrite(aof,cmd,sizeof(cmd)-1) == 0) goto werr; if (rioWriteBulkObject(aof,&key) == 0) goto werr; if (rioWrite(aof,subExpire.subkey(),sdslen(subExpire.subkey())) == 0) goto werr; diff --git a/src/cron.cpp b/src/cron.cpp index 230cf4ed4..9ef8bbfb4 100644 --- a/src/cron.cpp +++ b/src/cron.cpp @@ -70,6 +70,7 @@ void cronCommand(client *c) decrRefCount(o); // use an expire to trigger execution. Note: We use a subkey expire here so legacy clients don't delete it. setExpire(c, c->db, c->argv[ARG_NAME], c->argv[ARG_NAME], base + interval); + ++g_pserver->dirty; addReply(c, shared.ok); } @@ -86,6 +87,7 @@ void executeCronJobExpireHook(const char *key, robj *o) serverAssert(cFake->argc == 0); // Setup the args for the EVAL command + cFake->cmd = lookupCommandByCString("EVAL"); cFake->argc = 3 + job->veckeys.size() + job->vecargs.size(); cFake->argv = (robj**)zmalloc(sizeof(robj*) * cFake->argc, MALLOC_LOCAL); cFake->argv[0] = createStringObject("EVAL", 4); @@ -96,7 +98,17 @@ void executeCronJobExpireHook(const char *key, robj *o) for (size_t i = 0; i < job->vecargs.size(); ++i) cFake->argv[3+job->veckeys.size()+i] = createStringObject(job->vecargs[i].get(), job->vecargs[i].size()); + int lua_replicate_backup = g_pserver->lua_always_replicate_commands; + g_pserver->lua_always_replicate_commands = 0; evalCommand(cFake); + g_pserver->lua_always_replicate_commands = lua_replicate_backup; + + if (g_pserver->aof_state != AOF_OFF) + feedAppendOnlyFile(cFake->cmd,cFake->db->id,cFake->argv,cFake->argc); + // Active replicas do their own expiries, do not propogate + if (!g_pserver->fActiveReplica) + replicationFeedSlaves(g_pserver->slaves,cFake->db->id,cFake->argv,cFake->argc); + resetClient(cFake); robj *keyobj = createStringObject(key,sdslen(key)); diff --git a/src/db.cpp b/src/db.cpp index 562485941..5ecd3253c 100644 --- a/src/db.cpp +++ b/src/db.cpp @@ -1372,6 +1372,7 @@ expireEntry *getExpire(redisDb *db, robj_roptr key) { * keys. */ void propagateExpire(redisDb *db, robj *key, int lazy) { serverAssert(GlobalLocksAcquired()); + robj *argv[2]; argv[0] = lazy ? shared.unlink : shared.del; @@ -1389,6 +1390,48 @@ void propagateExpire(redisDb *db, robj *key, int lazy) { decrRefCount(argv[1]); } +void propagateSubkeyExpire(redisDb *db, int type, robj *key, robj *subkey) +{ + robj *argv[3]; + robj objT; + redisCommand *cmd = nullptr; + switch (type) + { + case OBJ_SET: + argv[0] = shared.srem; + argv[1] = key; + argv[2] = subkey; + cmd = cserver.sremCommand; + break; + + case OBJ_HASH: + argv[0] = shared.hdel; + argv[1] = key; + argv[2] = subkey; + cmd = cserver.hdelCommand; + break; + + case OBJ_ZSET: + argv[0] = shared.zrem; + argv[1] = key; + argv[2] = subkey; + cmd = cserver.zremCommand; + break; + + case OBJ_CRON: + return; // CRON jobs replicate in their own handler + + default: + serverPanic("Unknown subkey type"); + } + + if (g_pserver->aof_state != AOF_OFF) + feedAppendOnlyFile(cmd,db->id,argv,3); + // Active replicas do their own expiries, do not propogate + if (!g_pserver->fActiveReplica) + replicationFeedSlaves(g_pserver->slaves,db->id,argv,3); +} + /* Check if the key is expired. Note, this does not check subexpires */ int keyIsExpired(redisDb *db, robj *key) { expireEntry *pexpire = getExpire(db,key); diff --git a/src/debug.cpp b/src/debug.cpp index 1916dc79b..ca4da2d0a 100644 --- a/src/debug.cpp +++ b/src/debug.cpp @@ -31,6 +31,7 @@ #include "server.h" #include "sha1.h" /* SHA1 is used for DEBUG DIGEST */ #include "crc64.h" +#include "cron.h" #include #include @@ -251,6 +252,10 @@ void xorObjectDigest(redisDb *db, robj_roptr keyobj, unsigned char *digest, robj mt->digest(&md,mv->value); xorDigest(digest,md.x,sizeof(md.x)); } + } else if (o->type == OBJ_CRON) { + cronjob *job = (cronjob*)ptrFromObj(o); + mixDigest(digest, &job->interval, sizeof(job->interval)); + mixDigest(digest, job->script.get(), job->script.size()); } else { serverPanic("Unknown object type"); } diff --git a/src/expire.cpp b/src/expire.cpp index 1ac6ab415..ae9c75e8a 100644 --- a/src/expire.cpp +++ b/src/expire.cpp @@ -78,6 +78,10 @@ void activeExpireCycleExpire(redisDb *db, expireEntry &e, long long now) { dictEntry *de = dictFind(db->pdict, e.key()); robj *val = (robj*)dictGetVal(de); int deleted = 0; + + robj objKey; + initStaticStringObject(objKey, (char*)e.key()); + while (!pfat->FEmpty()) { if (pfat->nextExpireEntry().when > now) @@ -130,6 +134,11 @@ void activeExpireCycleExpire(redisDb *db, expireEntry &e, long long now) { default: serverAssert(false); } + + robj objSubkey; + initStaticStringObject(objSubkey, (char*)pfat->nextExpireEntry().spsubkey.get()); + propagateSubkeyExpire(db, val->type, &objKey, &objSubkey); + pfat->popfrontExpireEntry(); } @@ -144,22 +153,18 @@ void activeExpireCycleExpire(redisDb *db, expireEntry &e, long long now) { db->setexpire->insert(eT); } - robj objT; switch (val->type) { case OBJ_SET: - initStaticStringObject(objT, (char*)e.key()); - signalModifiedKey(db,&objT); - notifyKeyspaceEvent(NOTIFY_SET,"srem",&objT,db->id); + signalModifiedKey(db,&objKey); + notifyKeyspaceEvent(NOTIFY_SET,"srem",&objKey,db->id); break; } } if (pfat->FEmpty()) { - robj *keyobj = createStringObject(e.key(),sdslen(e.key())); - removeExpire(db, keyobj); - decrRefCount(keyobj); + removeExpire(db, &objKey); } } @@ -224,7 +229,8 @@ void expireMemberCore(client *c, robj *key, robj *subkey, long long basetime, lo } setExpire(c, c->db, key, subkey, when); - + signalModifiedKey(c->db, key); + g_pserver->dirty++; addReply(c, shared.cone); } @@ -256,6 +262,14 @@ void expireMemberAtCommand(client *c) expireMemberCore(c, c->argv[1], c->argv[2], 0, when, UNIT_SECONDS); } +void pexpireMemberAtCommand(client *c) +{ + long long when; + if (getLongLongFromObjectOrReply(c, c->argv[3], &when, NULL) != C_OK) + return; + + expireMemberCore(c, c->argv[1], c->argv[2], 0, when, UNIT_MILLISECONDS); +} /* Try to expire a few timed out keys. The algorithm used is adaptive and * will use few CPU cycles if there are few expiring keys, otherwise diff --git a/src/replication.cpp b/src/replication.cpp index dcd6a915b..a0a69da26 100644 --- a/src/replication.cpp +++ b/src/replication.cpp @@ -251,6 +251,8 @@ void feedReplicationBacklogWithObject(robj *o) { feedReplicationBacklog(p,len); } +sds catCommandForAofAndActiveReplication(sds buf, struct redisCommand *cmd, robj **argv, int argc); + void replicationFeedSlave(client *replica, int dictid, robj **argv, int argc, bool fSendRaw) { char llstr[LONG_STR_SIZE]; @@ -289,13 +291,23 @@ void replicationFeedSlave(client *replica, int dictid, robj **argv, int argc, bo * are queued in the output buffer until the initial SYNC completes), * or are already in sync with the master. */ - /* Add the multi bulk length. */ - addReplyArrayLenAsync(replica,argc); + if (fSendRaw) + { + /* Add the multi bulk length. */ + addReplyArrayLenAsync(replica,argc); - /* Finally any additional argument that was not stored inside the - * static buffer if any (from j to argc). */ - for (int j = 0; j < argc; j++) - addReplyBulkAsync(replica,argv[j]); + /* Finally any additional argument that was not stored inside the + * static buffer if any (from j to argc). */ + for (int j = 0; j < argc; j++) + addReplyBulkAsync(replica,argv[j]); + } + else + { + struct redisCommand *cmd = lookupCommand(szFromObj(argv[0])); + sds buf = catCommandForAofAndActiveReplication(sdsempty(), cmd, argv, argc); + addReplyProtoAsync(replica, buf, sdslen(buf)); + sdsfree(buf); + } } /* Propagate write commands to slaves, and populate the replication backlog diff --git a/src/server.cpp b/src/server.cpp index 88f0b539b..4efe103ab 100644 --- a/src/server.cpp +++ b/src/server.cpp @@ -634,6 +634,10 @@ struct redisCommand redisCommandTable[] = { {"expirememberat", expireMemberAtCommand, 4, "write fast @keyspace", 0,NULL,1,1,1,0,0,0}, + + {"pexpirememberat", pexpireMemberAtCommand, 4, + "write fast @keyspace", + 0,NULL,1,1,1,0,0,0}, {"pexpire",pexpireCommand,3, "write fast @keyspace", @@ -2289,6 +2293,9 @@ void createSharedObjects(void) { shared.rpoplpush = makeObjectShared(createStringObject("RPOPLPUSH",9)); shared.zpopmin = makeObjectShared(createStringObject("ZPOPMIN",7)); shared.zpopmax = makeObjectShared(createStringObject("ZPOPMAX",7)); + shared.hdel = makeObjectShared(createStringObject("HDEL", 4)); + shared.zrem = makeObjectShared(createStringObject("ZREM", 4)); + shared.srem = makeObjectShared(createStringObject("SREM", 4)); for (j = 0; j < OBJ_SHARED_INTEGERS; j++) { shared.integers[j] = makeObjectShared(createObject(OBJ_STRING,(void*)(long)j)); @@ -2514,6 +2521,8 @@ void initServerConfig(void) { cserver.xclaimCommand = lookupCommandByCString("xclaim"); cserver.xgroupCommand = lookupCommandByCString("xgroup"); cserver.rreplayCommand = lookupCommandByCString("rreplay"); + cserver.hdelCommand = lookupCommandByCString("hdel"); + cserver.zremCommand = lookupCommandByCString("zrem"); /* Slow log */ g_pserver->slowlog_log_slower_than = CONFIG_DEFAULT_SLOWLOG_LOG_SLOWER_THAN; diff --git a/src/server.h b/src/server.h index 218982bda..307321bab 100644 --- a/src/server.h +++ b/src/server.h @@ -1360,7 +1360,7 @@ struct sharedObjectsStruct { *masterdownerr, *roslaveerr, *execaborterr, *noautherr, *noreplicaserr, *busykeyerr, *oomerr, *plus, *messagebulk, *pmessagebulk, *subscribebulk, *unsubscribebulk, *psubscribebulk, *punsubscribebulk, *del, *unlink, - *rpop, *lpop, *lpush, *rpoplpush, *zpopmin, *zpopmax, *emptyscan, + *rpop, *lpop, *lpush, *rpoplpush, *zpopmin, *zpopmax, *emptyscan, *srem, *hdel, *zrem, *select[PROTO_SHARED_SELECT_CMDS], *integers[OBJ_SHARED_INTEGERS], *mbulkhdr[OBJ_SHARED_BULKHDR_LEN], /* "*\r\n" */ @@ -1581,7 +1581,7 @@ struct redisServerConst { *lpopCommand, *rpopCommand, *zpopminCommand, *zpopmaxCommand, *sremCommand, *execCommand, *expireCommand, *pexpireCommand, *xclaimCommand, - *xgroupCommand, *rreplayCommand; + *xgroupCommand, *rreplayCommand, *hdelCommand, *zremCommand; /* Configuration */ char *default_masteruser; /* AUTH with this user and masterauth with master */ @@ -2536,6 +2536,7 @@ int removeExpire(redisDb *db, robj *key); int removeExpireCore(redisDb *db, robj *key, dictEntry *de); int removeSubkeyExpire(redisDb *db, robj *key, robj *subkey); void propagateExpire(redisDb *db, robj *key, int lazy); +void propagateSubkeyExpire(redisDb *db, int type, robj *key, robj *subkey); int expireIfNeeded(redisDb *db, robj *key); expireEntry *getExpire(redisDb *db, robj_roptr key); void setExpire(client *c, redisDb *db, robj *key, robj *subkey, long long when); @@ -2659,6 +2660,8 @@ extern "C" char *redisGitSHA1(void); extern "C" char *redisGitDirty(void); extern "C" uint64_t redisBuildId(void); +int parseUnitString(const char *sz); + /* Commands prototypes */ void authCommand(client *c); void pingCommand(client *c); @@ -2736,6 +2739,7 @@ void expireCommand(client *c); void expireatCommand(client *c); void expireMemberCommand(client *c); void expireMemberAtCommand(client *c); +void pexpireMemberAtCommand(client *c); void pexpireCommand(client *c); void pexpireatCommand(client *c); void getsetCommand(client *c); diff --git a/tests/integration/aof.tcl b/tests/integration/aof.tcl index 277b0d3df..0a1a21fdd 100644 --- a/tests/integration/aof.tcl +++ b/tests/integration/aof.tcl @@ -257,4 +257,26 @@ tags {"aof"} { r expire x -1 } } + + ## Test that PEXPIREMEMBERAT is loaded correctly + create_aof { + append_to_aof [formatCommand sadd testkey a b c d] + append_to_aof [formatCommand pexpirememberat testkey a 1000] + } + + start_server_aof [list dir $server_path aof-load-truncated no] { + test "AOF+EXPIREMEMBER: Server shuold have been started" { + assert_equal 1 [is_alive $srv] + } + + test "AOF+PEXPIREMEMBERAT: set should have 3 values" { + set client [redis [dict get $srv host] [dict get $srv port]] + wait_for_condition 50 100 { + [catch {$client ping} e] == 0 + } else { + fail "Loading DB is taking too much time." + } + assert_equal 3 [$client scard testkey] + } + } } diff --git a/tests/integration/replication-4.tcl b/tests/integration/replication-4.tcl index 3c6df52a8..4065de786 100644 --- a/tests/integration/replication-4.tcl +++ b/tests/integration/replication-4.tcl @@ -151,5 +151,93 @@ start_server {tags {"repl"}} { fail "SPOP replication inconsistency" } } + + test {Replication of EXPIREMEMBER (set) command} { + $master sadd testkey a b c d + wait_for_condition 50 100 { + [$master debug digest] eq [$slave debug digest] + } else { + fail "Failed to replicate set" + } + $master expiremember testkey a 1 + after 1000 + wait_for_condition 50 100 { + [$master scard testkey] eq 3 + } else { + fail "expiremember failed to work on master" + } + wait_for_condition 50 100 { + [$slave scard testkey] eq 3 + } else { + assert_equal [$slave scard testkey] 3 + } + $master del testkey + } + + test {Replication of EXPIREMEMBER (hash) command} { + $master hset testkey a value + $master hset testkey b value + wait_for_condition 50 100 { + [$master debug digest] eq [$slave debug digest] + } else { + fail "Failed to replicate set" + } + $master expiremember testkey a 1 + after 1000 + wait_for_condition 50 100 { + [$master hlen testkey] eq 1 + } else { + fail "expiremember failed to work on master" + } + wait_for_condition 50 100 { + [$slave hlen testkey] eq 1 + } else { + assert_equal [$slave hlen testkey] 1 + } + $master del testkey + } + + test {Replication of EXPIREMEMBER (zset) command} { + $master zadd testkey 1 a + $master zadd testkey 2 b + wait_for_condition 50 100 { + [$master debug digest] eq [$slave debug digest] + } else { + fail "Failed to replicate set" + } + $master expiremember testkey a 1 + after 1000 + wait_for_condition 50 100 { + [$master zcard testkey] eq 1 + } else { + fail "expiremember failed to work on master" + } + wait_for_condition 50 100 { + [$slave zcard testkey] eq 1 + } else { + assert_equal [$slave zcard testkey] 1 + } + } + + test {keydb.cron replicates} { + $master del testkey + $master keydb.cron testjob repeat 0 1000000 {redis.call("incr", "testkey")} 1 testkey + after 300 + assert_equal 1 [$master get testkey] + assert_equal 1 [$master exists testjob] + + wait_for_condition 50 100 { + [$master debug digest] eq [$slave debug digest] + } else { + fail "KEYDB.CRON failed to replicate" + } + $master del testjob + $master del testkey + wait_for_condition 50 1000 { + [$master debug digest] eq [$slave debug digest] + } else { + fail "cron delete failed to propogate" + } + } } } diff --git a/tests/integration/replication-active.tcl b/tests/integration/replication-active.tcl index 0c28eb85d..b1d1c5217 100644 --- a/tests/integration/replication-active.tcl +++ b/tests/integration/replication-active.tcl @@ -77,6 +77,94 @@ start_server {tags {"active-repl"} overrides {active-replica yes}} { $master flushall } + test {Replication of EXPIREMEMBER (set) command (Active)} { + $master sadd testkey a b c d + wait_for_condition 50 100 { + [$master debug digest] eq [$slave debug digest] + } else { + fail "Failed to replicate set" + } + $master expiremember testkey a 1 + after 1000 + wait_for_condition 50 100 { + [$master scard testkey] eq 3 + } else { + fail "expiremember failed to work on master" + } + wait_for_condition 50 100 { + [$slave scard testkey] eq 3 + } else { + assert_equal [$slave scard testkey] 3 + } + $master del testkey + } + + test {Replication of EXPIREMEMBER (hash) command (Active)} { + $master hset testkey a value + $master hset testkey b value + wait_for_condition 50 100 { + [$master debug digest] eq [$slave debug digest] + } else { + fail "Failed to replicate set" + } + $master expiremember testkey a 1 + after 1000 + wait_for_condition 50 100 { + [$master hlen testkey] eq 1 + } else { + fail "expiremember failed to work on master" + } + wait_for_condition 50 100 { + [$slave hlen testkey] eq 1 + } else { + assert_equal [$slave hlen testkey] 1 + } + $master del testkey + } + + test {Replication of EXPIREMEMBER (zset) command (Active)} { + $master zadd testkey 1 a + $master zadd testkey 2 b + wait_for_condition 50 100 { + [$master debug digest] eq [$slave debug digest] + } else { + fail "Failed to replicate set" + } + $master expiremember testkey a 1 + after 1000 + wait_for_condition 50 100 { + [$master zcard testkey] eq 1 + } else { + fail "expiremember failed to work on master" + } + wait_for_condition 50 100 { + [$slave zcard testkey] eq 1 + } else { + assert_equal [$slave zcard testkey] 1 + } + } + + test {keydb.cron replicates (Active) } { + $master del testkey + $master keydb.cron testjob repeat 0 1000000 {redis.call("incr", "testkey")} 1 testkey + after 300 + assert_equal 1 [$master get testkey] + assert_equal 1 [$master exists testjob] + + wait_for_condition 50 100 { + [$master debug digest] eq [$slave debug digest] + } else { + fail "KEYDB.CRON failed to replicate" + } + $master del testjob + $master del testkey + wait_for_condition 50 1000 { + [$master debug digest] eq [$slave debug digest] + } else { + fail "cron delete failed to propogate" + } + } + test {Active replicas WAIT} { # Test that wait succeeds since replicas should be syncronized $master set testkey foo @@ -113,37 +201,38 @@ start_server {tags {"active-repl"} overrides {active-replica yes}} { assert_equal {0} [$slave del testkey1] } - test {Active replica expire propogates when source is down} { - $slave flushall - $slave set testkey2 foo - $slave set testkey1 foo - wait_for_condition 50 1000 { - [string match *foo* [$master get testkey1]] - } else { - fail "Replication failed to propogate" - } - $slave expire testkey1 2 - assert_equal {1} [$slave wait 1 500] { "value should propogate - within 0.5 seconds" } - exec kill -SIGSTOP $slave_pid + test {Active replica expire propogates when source is down} { + $slave flushall + $slave set testkey2 foo + $slave set testkey1 foo + wait_for_condition 50 1000 { + [string match *foo* [$master get testkey1]] + } else { + fail "Replication failed to propogate" + } + $slave expire testkey1 2 + assert_equal {1} [$slave wait 1 500] { "value should propogate + within 0.5 seconds" } + exec kill -SIGSTOP $slave_pid after 3000 - # Ensure testkey1 is gone. Note, we can't do this directly as the normal commands lie to us - # about what is actually in the dict. The only way to know is with a count from info + # Ensure testkey1 is gone. Note, we can't do this directly as the normal commands lie to us + # about what is actually in the dict. The only way to know is with a count from info assert_equal {1} [expr [string first {keys=1} [$master info keyspace]] >= 0] {"slave expired"} - } - exec kill -SIGCONT $slave_pid + } - test {Active replica different databases} { - $master select 3 - $master set testkey abcd - $master select 2 - $master del testkey - $slave select 3 - wait_for_condition 50 1000 { - [string match abcd [$slave get testkey]] - } else { - fail "Replication failed to propogate DB 3" - } + exec kill -SIGCONT $slave_pid + + test {Active replica different databases} { + $master select 3 + $master set testkey abcd + $master select 2 + $master del testkey + $slave select 3 + wait_for_condition 50 1000 { + [string match abcd [$slave get testkey]] + } else { + fail "Replication failed to propogate DB 3" } } } +} From de11140a1716887b64e9200a1c84372399db8e90 Mon Sep 17 00:00:00 2001 From: John Sully Date: Sat, 4 Apr 2020 22:32:15 -0400 Subject: [PATCH 197/225] Role command protocol corruption with multiple masters Former-commit-id: 888d69a87a0076caa5b381d2531a6a638aa69051 --- src/replication.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/replication.cpp b/src/replication.cpp index a0a69da26..4c5e4a26c 100644 --- a/src/replication.cpp +++ b/src/replication.cpp @@ -2594,6 +2594,8 @@ void roleCommand(client *c) { listNode *ln; listRewind(g_pserver->masters, &li); + if (listLength(g_pserver->masters) > 1) + addReplyArrayLen(c,listLength(g_pserver->masters)); while ((ln = listNext(&li))) { redisMaster *mi = (redisMaster*)listNodeValue(ln); From 662a1b6a42c893357e8edf561f7aeeecd23282c7 Mon Sep 17 00:00:00 2001 From: John Sully Date: Sat, 4 Apr 2020 22:45:12 -0400 Subject: [PATCH 198/225] Fix issue #164 Former-commit-id: f112c77fcc3a60277ce344478bc37adb0fe4a99d --- src/evict.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/evict.cpp b/src/evict.cpp index 04fd7a7d6..f21ee0ea7 100644 --- a/src/evict.cpp +++ b/src/evict.cpp @@ -462,7 +462,7 @@ int freeMemoryIfNeeded(void) { serverAssert(GlobalLocksAcquired()); /* By default replicas should ignore maxmemory * and just be masters exact copies. */ - if (listLength(g_pserver->masters) && g_pserver->repl_slave_ignore_maxmemory) return C_OK; + if (listLength(g_pserver->masters) && g_pserver->repl_slave_ignore_maxmemory && !g_pserver->fActiveReplica) return C_OK; size_t mem_reported, mem_tofree, mem_freed; mstime_t latency, eviction_latency; From 6f15ce125cfe0c8a6ebe8d56b7fcb30d9169e7cf Mon Sep 17 00:00:00 2001 From: John Sully Date: Sat, 4 Apr 2020 22:58:17 -0400 Subject: [PATCH 199/225] Add the ability to set a starting core # when setting thread affinity Former-commit-id: 9e2e2067c6df5919f1c6b8b9e6e3457c7edc0755 --- src/config.cpp | 10 ++++++++-- src/networking.cpp | 2 +- src/server.cpp | 5 +++-- src/server.h | 1 + 4 files changed, 13 insertions(+), 5 deletions(-) diff --git a/src/config.cpp b/src/config.cpp index 4035ca376..70448a916 100644 --- a/src/config.cpp +++ b/src/config.cpp @@ -782,8 +782,14 @@ void loadServerConfigFromString(char *config) { } else if (strcasecmp(argv[1], "false") == 0) { cserver.fThreadAffinity = FALSE; } else { - err = "Unknown argument: server-thread-affinity expects either true or false"; - goto loaderr; + int offset = atoi(argv[1]); + if (offset > 0) { + cserver.fThreadAffinity = TRUE; + cserver.threadAffinityOffset = offset-1; + } else { + err = "Unknown argument: server-thread-affinity expects either true or false"; + goto loaderr; + } } } else if (!strcasecmp(argv[0], "active-replica") && argc == 2) { g_pserver->fActiveReplica = yesnotoi(argv[1]); diff --git a/src/networking.cpp b/src/networking.cpp index 276095715..fb7d286c6 100644 --- a/src/networking.cpp +++ b/src/networking.cpp @@ -1073,7 +1073,7 @@ static void acceptCommonHandler(int fd, int flags, char *ip, int iel) { // Set thread affinity if (cserver.fThreadAffinity) { - int cpu = iel; + int cpu = iel + cserver.threadAffinityOffset; if (setsockopt(fd, SOL_SOCKET, SO_INCOMING_CPU, &cpu, sizeof(iel)) != 0) { serverLog(LL_WARNING, "Failed to set socket affinity"); diff --git a/src/server.cpp b/src/server.cpp index 4efe103ab..0c4b3116c 100644 --- a/src/server.cpp +++ b/src/server.cpp @@ -2547,6 +2547,7 @@ void initServerConfig(void) { /* Multithreading */ cserver.cthreads = CONFIG_DEFAULT_THREADS; cserver.fThreadAffinity = CONFIG_DEFAULT_THREAD_AFFINITY; + cserver.threadAffinityOffset = 0; } extern char **environ; @@ -5350,10 +5351,10 @@ int main(int argc, char **argv) { #ifdef __linux__ cpu_set_t cpuset; CPU_ZERO(&cpuset); - CPU_SET(iel, &cpuset); + CPU_SET(iel + cserver.threadAffinityOffset, &cpuset); if (pthread_setaffinity_np(rgthread[iel], sizeof(cpu_set_t), &cpuset) == 0) { - serverLog(LOG_INFO, "Binding thread %d to cpu %d", iel, iel); + serverLog(LOG_INFO, "Binding thread %d to cpu %d", iel, iel + cserver.threadAffinityOffset + 1); } #else serverLog(LL_WARNING, "CPU pinning not available on this platform"); diff --git a/src/server.h b/src/server.h index 307321bab..5d5f4b3e0 100644 --- a/src/server.h +++ b/src/server.h @@ -1574,6 +1574,7 @@ struct redisServerConst { int cthreads; /* Number of main worker threads */ int fThreadAffinity; /* Should we pin threads to cores? */ + int threadAffinityOffset = 0; /* Where should we start pinning them? */ char *pidfile; /* PID file path */ /* Fast pointers to often looked up command */ From d683a62b433e04efc74ec36fa8cf783f6be9ddd9 Mon Sep 17 00:00:00 2001 From: John Sully Date: Mon, 13 Apr 2020 22:11:34 -0400 Subject: [PATCH 200/225] Optimize replicaFeedSlaves by removing use of snprintf Former-commit-id: 32561a99124542461de283d5035f869b5fc9bc2f --- src/replication.cpp | 46 ++++++++++++++++++++++++++++++++++----------- 1 file changed, 35 insertions(+), 11 deletions(-) diff --git a/src/replication.cpp b/src/replication.cpp index 4c5e4a26c..7a5fce456 100644 --- a/src/replication.cpp +++ b/src/replication.cpp @@ -310,6 +310,26 @@ void replicationFeedSlave(client *replica, int dictid, robj **argv, int argc, bo } } +static int writeProtoNum(char *dst, const size_t cchdst, long long num) +{ + if (cchdst < 1) + return 0; + dst[0] = '$'; + int cch = 1; + cch += ll2string(dst + cch, cchdst - cch, digits10(num)); + int chCpyT = std::min(cchdst - cch, 2); + memcpy(dst + cch, "\r\n", chCpyT); + cch += chCpyT; + cch += ll2string(dst + cch, cchdst-cch, num); + chCpyT = std::min(cchdst - cch, 3); + memcpy(dst + cch, "\r\n", chCpyT); + if (chCpyT == 3) + cch += 2; + else + cch += chCpyT; + return cch; +} + /* Propagate write commands to slaves, and populate the replication backlog * as well. This function is used if the instance is a master: we use * the commands received by our clients in order to create the replication @@ -354,24 +374,28 @@ void replicationFeedSlaves(list *slaves, int dictid, robj **argv, int argc) { serverAssert(argc > 0); serverAssert(cchbuf > 0); - char uuid[40] = {'\0'}; + char uuid[37]; uuid_unparse(cserver.uuid, uuid); - char proto[1024]; - int cchProto = snprintf(proto, sizeof(proto), "*5\r\n$7\r\nRREPLAY\r\n$%d\r\n%s\r\n$%lld\r\n", (int)strlen(uuid), uuid, cchbuf); - cchProto = std::min((int)sizeof(proto), cchProto); + + + // The code below used to be: snprintf(proto, sizeof(proto), "*5\r\n$7\r\nRREPLAY\r\n$%d\r\n%s\r\n$%lld\r\n", (int)strlen(uuid), uuid, cchbuf); + // but that was much too slow + char proto[1024] = "*5\r\n$7\r\nRREPLAY\r\n$36\r\n00000000-0000-0000-0000-000000000000\r\n$"; + int cchProto = strlen(proto); + memcpy(proto + 22, uuid, 36); // Note UUID_STR_LEN includes the \0 trailing byte which we don't want + cchProto += ll2string(proto + cchProto, sizeof(proto)-cchProto, cchbuf); + memcpy(proto + cchProto, "\r\n", 3); + cchProto += 2; + long long master_repl_offset_start = g_pserver->master_repl_offset; char szDbNum[128]; - int cchDictIdNum = snprintf(szDbNum, sizeof(szDbNum), "%d", dictid); - int cchDbNum = snprintf(szDbNum, sizeof(szDbNum), "$%d\r\n%d\r\n", cchDictIdNum, dictid); - cchDbNum = std::min(cchDbNum, sizeof(szDbNum)); // snprintf is tricky like that + int cchDbNum = writeProtoNum(szDbNum, sizeof(szDbNum), dictid); + char szMvcc[128]; incrementMvccTstamp(); - uint64_t mvccTstamp = getMvccTstamp(); - int cchMvccNum = snprintf(szMvcc, sizeof(szMvcc), "%" PRIu64, mvccTstamp); - int cchMvcc = snprintf(szMvcc, sizeof(szMvcc), "$%d\r\n%" PRIu64 "\r\n", cchMvccNum, mvccTstamp); - cchMvcc = std::min(cchMvcc, sizeof(szMvcc)); // tricky snprintf + int cchMvcc = writeProtoNum(szMvcc, sizeof(szMvcc), getMvccTstamp()); /* Write the command to the replication backlog if any. */ if (g_pserver->repl_backlog) From 2e4e4f4134c1f4555c46748427b96112c0779de5 Mon Sep 17 00:00:00 2001 From: John Sully Date: Mon, 13 Apr 2020 22:35:06 -0400 Subject: [PATCH 201/225] Don't do active replica work if we're not an active replica Former-commit-id: 63dd1fb599cfe959c0298825ed56ab06335b3fd7 --- src/replication.cpp | 30 +++++++++++++++++++++--------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/src/replication.cpp b/src/replication.cpp index 7a5fce456..8380862ca 100644 --- a/src/replication.cpp +++ b/src/replication.cpp @@ -380,22 +380,34 @@ void replicationFeedSlaves(list *slaves, int dictid, robj **argv, int argc) { // The code below used to be: snprintf(proto, sizeof(proto), "*5\r\n$7\r\nRREPLAY\r\n$%d\r\n%s\r\n$%lld\r\n", (int)strlen(uuid), uuid, cchbuf); // but that was much too slow - char proto[1024] = "*5\r\n$7\r\nRREPLAY\r\n$36\r\n00000000-0000-0000-0000-000000000000\r\n$"; - int cchProto = strlen(proto); - memcpy(proto + 22, uuid, 36); // Note UUID_STR_LEN includes the \0 trailing byte which we don't want - cchProto += ll2string(proto + cchProto, sizeof(proto)-cchProto, cchbuf); - memcpy(proto + cchProto, "\r\n", 3); - cchProto += 2; + static const char *protoRREPLAY = "*5\r\n$7\r\nRREPLAY\r\n$36\r\n00000000-0000-0000-0000-000000000000\r\n$"; + char proto[1024]; + int cchProto = 0; + if (!fSendRaw) + { + cchProto = strlen(protoRREPLAY); + memcpy(proto, protoRREPLAY, strlen(protoRREPLAY)); + memcpy(proto + 22, uuid, 36); // Note UUID_STR_LEN includes the \0 trailing byte which we don't want + cchProto += ll2string(proto + cchProto, sizeof(proto)-cchProto, cchbuf); + memcpy(proto + cchProto, "\r\n", 3); + cchProto += 2; + } long long master_repl_offset_start = g_pserver->master_repl_offset; char szDbNum[128]; - int cchDbNum = writeProtoNum(szDbNum, sizeof(szDbNum), dictid); + int cchDbNum = 0; + if (!fSendRaw) + cchDbNum = writeProtoNum(szDbNum, sizeof(szDbNum), dictid); char szMvcc[128]; - incrementMvccTstamp(); - int cchMvcc = writeProtoNum(szMvcc, sizeof(szMvcc), getMvccTstamp()); + int cchMvcc = 0; + incrementMvccTstamp(); // Always increment MVCC tstamp so we're consistent with active and normal replication + if (!fSendRaw) + { + cchMvcc = writeProtoNum(szMvcc, sizeof(szMvcc), getMvccTstamp()); + } /* Write the command to the replication backlog if any. */ if (g_pserver->repl_backlog) From 36c9a2c897549e2767115cf640e6f7195e0ed1c5 Mon Sep 17 00:00:00 2001 From: John Sully Date: Mon, 13 Apr 2020 22:45:15 -0400 Subject: [PATCH 202/225] Cache fake client in replicaFeedSlaves Former-commit-id: 8e81e5f29e718395b32a60ff263808305d0b5818 --- src/replication.cpp | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/src/replication.cpp b/src/replication.cpp index 8380862ca..179fca5a4 100644 --- a/src/replication.cpp +++ b/src/replication.cpp @@ -340,6 +340,8 @@ void replicationFeedSlaves(list *slaves, int dictid, robj **argv, int argc) { listIter li, liReply; int j, len; serverAssert(GlobalLocksAcquired()); + static client *fake = nullptr; + if (dictid < 0) dictid = 0; // this can happen if we send a PING before any real operation @@ -357,8 +359,12 @@ void replicationFeedSlaves(list *slaves, int dictid, robj **argv, int argc) { /* We can't have slaves attached and no backlog. */ serverAssert(!(listLength(slaves) != 0 && g_pserver->repl_backlog == NULL)); - client *fake = createClient(-1, serverTL - g_pserver->rgthreadvar); - fake->flags |= CLIENT_FORCE_REPLY; + if (fake == nullptr) + { + fake = createClient(-1, serverTL - g_pserver->rgthreadvar); + fake->flags |= CLIENT_FORCE_REPLY; + } + bool fSendRaw = !g_pserver->fActiveReplica; replicationFeedSlave(fake, dictid, argv, argc, fSendRaw); // Note: updates the repl log, keep above the repl update code below @@ -374,10 +380,6 @@ void replicationFeedSlaves(list *slaves, int dictid, robj **argv, int argc) { serverAssert(argc > 0); serverAssert(cchbuf > 0); - char uuid[37]; - uuid_unparse(cserver.uuid, uuid); - - // The code below used to be: snprintf(proto, sizeof(proto), "*5\r\n$7\r\nRREPLAY\r\n$%d\r\n%s\r\n$%lld\r\n", (int)strlen(uuid), uuid, cchbuf); // but that was much too slow static const char *protoRREPLAY = "*5\r\n$7\r\nRREPLAY\r\n$36\r\n00000000-0000-0000-0000-000000000000\r\n$"; @@ -385,8 +387,11 @@ void replicationFeedSlaves(list *slaves, int dictid, robj **argv, int argc) { int cchProto = 0; if (!fSendRaw) { + char uuid[37]; + uuid_unparse(cserver.uuid, uuid); + cchProto = strlen(protoRREPLAY); - memcpy(proto, protoRREPLAY, strlen(protoRREPLAY)); + memcpy(proto, protoRREPLAY, strlen(protoRREPLAY)); memcpy(proto + 22, uuid, 36); // Note UUID_STR_LEN includes the \0 trailing byte which we don't want cchProto += ll2string(proto + cchProto, sizeof(proto)-cchProto, cchbuf); memcpy(proto + cchProto, "\r\n", 3); @@ -405,9 +410,7 @@ void replicationFeedSlaves(list *slaves, int dictid, robj **argv, int argc) { int cchMvcc = 0; incrementMvccTstamp(); // Always increment MVCC tstamp so we're consistent with active and normal replication if (!fSendRaw) - { cchMvcc = writeProtoNum(szMvcc, sizeof(szMvcc), getMvccTstamp()); - } /* Write the command to the replication backlog if any. */ if (g_pserver->repl_backlog) @@ -492,7 +495,11 @@ void replicationFeedSlaves(list *slaves, int dictid, robj **argv, int argc) { } } - freeClient(fake); + // Cleanup cached fake client output buffers + fake->bufpos = 0; + fake->sentlen = 0; + fake->reply_bytes = 0; + listEmpty(fake->reply); } /* This function is used in order to proxy what we receive from our master From 234e6e7b0940db9484e99dd8180dfd18ed61c046 Mon Sep 17 00:00:00 2001 From: Guy Benoish Date: Mon, 3 Feb 2020 17:19:00 +0530 Subject: [PATCH 203/225] Diskless-load emptyDb-related fixes 1. Call emptyDb even in case of diskless-load: We want modules to get the same FLUSHDB event as disk-based replication. 2. Do not fire any module events when flushing the backups array. 3. Delete redundant call to signalFlushedDb (Called from emptyDb). Former-commit-id: aa8a3077a2d20e66e34f72f2860d0cc3daad496e --- src/db.cpp | 55 +++++++++++++++++++++++++++------------------ src/replication.cpp | 14 +++++++----- src/server.h | 1 + 3 files changed, 43 insertions(+), 27 deletions(-) diff --git a/src/db.cpp b/src/db.cpp index 2a1df37ac..b1d9cad8e 100644 --- a/src/db.cpp +++ b/src/db.cpp @@ -443,7 +443,10 @@ robj *dbUnshareStringValue(redisDb *db, robj *key, robj *o) { * DB number if we want to flush only a single Redis database number. * * Flags are be EMPTYDB_NO_FLAGS if no special flags are specified or - * EMPTYDB_ASYNC if we want the memory to be freed in a different thread + * 1. EMPTYDB_ASYNC if we want the memory to be freed in a different thread. + * 2. EMPTYDB_BACKUP if we want to empty the backup dictionaries created by + * disklessLoadMakeBackups. In that case we only free memory and avoid + * firing module events. * and the function to return ASAP. * * On success the fuction returns the number of keys removed from the @@ -451,6 +454,8 @@ robj *dbUnshareStringValue(redisDb *db, robj *key, robj *o) { * DB number is out of range, and errno is set to EINVAL. */ long long emptyDbGeneric(redisDb *dbarray, int dbnum, int flags, void(callback)(void*)) { int async = (flags & EMPTYDB_ASYNC); + int backup = (flags & EMPTYDB_BACKUP); /* Just free the memory, nothing else */ + RedisModuleFlushInfoV1 fi = {REDISMODULE_FLUSHINFO_VERSION,!async,dbnum}; long long removed = 0; if (dbnum < -1 || dbnum >= cserver.dbnum) { @@ -458,16 +463,18 @@ long long emptyDbGeneric(redisDb *dbarray, int dbnum, int flags, void(callback)( return -1; } - /* Fire the flushdb modules event. */ - RedisModuleFlushInfoV1 fi = {REDISMODULE_FLUSHINFO_VERSION,!async,dbnum}; - moduleFireServerEvent(REDISMODULE_EVENT_FLUSHDB, - REDISMODULE_SUBEVENT_FLUSHDB_START, - &fi); + /* Pre-flush actions */ + if (!backup) { + /* Fire the flushdb modules event. */ + moduleFireServerEvent(REDISMODULE_EVENT_FLUSHDB, + REDISMODULE_SUBEVENT_FLUSHDB_START, + &fi); - /* Make sure the WATCHed keys are affected by the FLUSH* commands. - * Note that we need to call the function while the keys are still - * there. */ - signalFlushedDb(dbnum); + /* Make sure the WATCHed keys are affected by the FLUSH* commands. + * Note that we need to call the function while the keys are still + * there. */ + signalFlushedDb(dbnum); + } int startdb, enddb; if (dbnum == -1) { @@ -486,20 +493,24 @@ long long emptyDbGeneric(redisDb *dbarray, int dbnum, int flags, void(callback)( dbarray[j].setexpire->clear(); } } - if (g_pserver->cluster_enabled) { - if (async) { - slotToKeyFlushAsync(); - } else { - slotToKeyFlush(); + + /* Post-flush actions */ + if (!backup) { + if (g_pserver->cluster_enabled) { + if (async) { + slotToKeyFlushAsync(); + } else { + slotToKeyFlush(); + } } - } - if (dbnum == -1) flushSlaveKeysWithExpireList(); + if (dbnum == -1) flushSlaveKeysWithExpireList(); - /* Also fire the end event. Note that this event will fire almost - * immediately after the start event if the flush is asynchronous. */ - moduleFireServerEvent(REDISMODULE_EVENT_FLUSHDB, - REDISMODULE_SUBEVENT_FLUSHDB_END, - &fi); + /* Also fire the end event. Note that this event will fire almost + * immediately after the start event if the flush is asynchronous. */ + moduleFireServerEvent(REDISMODULE_EVENT_FLUSHDB, + REDISMODULE_SUBEVENT_FLUSHDB_END, + &fi); + } return removed; } diff --git a/src/replication.cpp b/src/replication.cpp index aa220c95a..11ac8eb4e 100644 --- a/src/replication.cpp +++ b/src/replication.cpp @@ -1663,8 +1663,8 @@ void disklessLoadRestoreBackups(redisDb *backup, int restore, int empty_db_flags g_pserver->db[i] = backup[i]; } } else { - /* Delete. */ - emptyDbGeneric(backup,-1,empty_db_flags,replicationEmptyDbCallback); + /* Delete (Pass EMPTYDB_BACKUP in order to avoid firing module events) . */ + emptyDbGeneric(backup,-1,empty_db_flags|EMPTYDB_BACKUP,replicationEmptyDbCallback); for (int i=0; iaof_state != AOF_OFF) stopAppendOnly(); - signalFlushedDb(-1); /* When diskless RDB loading is used by replicas, it may be configured * in order to save the current DB instead of throwing it away, @@ -1863,10 +1862,15 @@ void readSyncBulkPayload(connection *conn) { if (use_diskless_load && g_pserver->repl_diskless_load == REPL_DISKLESS_LOAD_SWAPDB) { + /* Create a backup of server.db[] and initialize to empty + * dictionaries */ diskless_load_backup = disklessLoadMakeBackups(); - } else { - emptyDb(-1,empty_db_flags,replicationEmptyDbCallback); } + /* We call to emptyDb even in case of REPL_DISKLESS_LOAD_SWAPDB + * (Where disklessLoadMakeBackups left server.db empty) because we + * want to execute all the auxiliary logic of emptyDb (Namely, + * fire module events) */ + emptyDb(-1,empty_db_flags,replicationEmptyDbCallback); /* Before loading the DB into memory we need to delete the readable * handler, otherwise it will get called recursively since diff --git a/src/server.h b/src/server.h index 957067bfa..b8a11e888 100644 --- a/src/server.h +++ b/src/server.h @@ -2627,6 +2627,7 @@ robj *dbUnshareStringValue(redisDb *db, robj *key, robj *o); #define EMPTYDB_NO_FLAGS 0 /* No flags. */ #define EMPTYDB_ASYNC (1<<0) /* Reclaim memory in another thread. */ +#define EMPTYDB_BACKUP (1<<2) /* DB array is a backup for REPL_DISKLESS_LOAD_SWAPDB. */ long long emptyDb(int dbnum, int flags, void(callback)(void*)); long long emptyDbGeneric(redisDb *dbarray, int dbnum, int flags, void(callback)(void*)); void flushAllDataAndResetRDB(int flags); From 1a078be8433521f752ef4a0bf7cb75d28fa8c2b3 Mon Sep 17 00:00:00 2001 From: Johannes Truschnigg Date: Thu, 19 Dec 2019 21:47:24 +0100 Subject: [PATCH 204/225] Signal systemd readiness atfer Partial Resync "Partial Resynchronization" is a special variant of replication success that we have to tell systemd about if it is managing redis-server via a Type=Notify service unit. Former-commit-id: dd9502f373eb7e32aee69a30dcb521bea3ccd3ad --- src/replication.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/replication.cpp b/src/replication.cpp index 11ac8eb4e..1edbb9b06 100644 --- a/src/replication.cpp +++ b/src/replication.cpp @@ -2584,6 +2584,10 @@ void syncWithMaster(connection *conn) { if (psync_result == PSYNC_CONTINUE) { serverLog(LL_NOTICE, "MASTER <-> REPLICA sync: Master accepted a Partial Resynchronization."); + if (server.supervised_mode == SUPERVISED_SYSTEMD) { + redisCommunicateSystemd("STATUS=MASTER <-> REPLICA sync: Partial Resynchronization accepted. Ready to accept connections.\n"); + redisCommunicateSystemd("READY=1\n"); + } return; } From 34f7c653d54e428bed3a6fc686f9612ccbbdd348 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 3 Mar 2020 14:58:11 +0100 Subject: [PATCH 205/225] Remove RDB files used for replication in persistence-less instances. Former-commit-id: b323645227a3e2cc5928e649586221aba508b10d --- src/replication.cpp | 49 ++++++++++++++++++++++++++++++++++++++++++++- src/server.cpp | 8 ++++++++ src/server.h | 1 + 3 files changed, 57 insertions(+), 1 deletion(-) diff --git a/src/replication.cpp b/src/replication.cpp index 1edbb9b06..3d0cafc56 100644 --- a/src/replication.cpp +++ b/src/replication.cpp @@ -53,6 +53,11 @@ void putSlaveOnline(client *replica); int cancelReplicationHandshake(redisMaster *mi); static void propagateMasterStaleKeys(); +/* We take a global flag to remember if this instance generated an RDB + * because of replication, so that we can remove the RDB file in case + * the instance is configured to have no persistence. */ +int RDBGeneratedByReplication = 0; + /* --------------------------- Utility functions ---------------------------- */ /* Return the pointer to a string representing the replica ip:listening_port @@ -789,6 +794,10 @@ int startBgsaveForReplication(int mincapa) { retval = C_ERR; } + /* If we succeeded to start a BGSAVE with disk target, let's remember + * this fact, so that we can later delete the file if needed. */ + if (retval == C_OK && !socket_target) RDBGeneratedByReplication = 1; + /* If we failed to BGSAVE, remove the slaves waiting for a full * resynchronization from the list of slaves, inform them with * an error about what happened, close the connection ASAP. */ @@ -1132,6 +1141,36 @@ void putSlaveOnline(client *replica) { } } +/* We call this function periodically to remove an RDB file that was + * generated because of replication, in an instance that is otherwise + * without any persistence. We don't want instances without persistence + * to take RDB files around, this violates certain policies in certain + * environments. */ +void removeRDBUsedToSyncReplicas(void) { + if (allPersistenceDisabled() && RDBGeneratedByReplication) { + client *slave; + listNode *ln; + listIter li; + + int delrdb = 1; + listRewind(server.slaves,&li); + while((ln = listNext(&li))) { + slave = ln->value; + if (slave->replstate == SLAVE_STATE_WAIT_BGSAVE_START || + slave->replstate == SLAVE_STATE_WAIT_BGSAVE_END || + slave->replstate == SLAVE_STATE_SEND_BULK) + { + delrdb = 0; + break; /* No need to check the other replicas. */ + } + } + if (delrdb) { + RDBGeneratedByReplication = 0; + unlink(server.rdb_filename); + } + } +} + void sendBulkToSlave(connection *conn) { client *replica = (client*)connGetPrivateData(conn); serverAssert(FCorrectThread(replica)); @@ -1144,7 +1183,8 @@ void sendBulkToSlave(connection *conn) { if (replica->replpreamble) { nwritten = connWrite(conn,replica->replpreamble,sdslen(replica->replpreamble)); if (nwritten == -1) { - serverLog(LL_VERBOSE,"Write error sending RDB preamble to replica: %s", + serverLog(LL_VERBOSE, + "Write error sending RDB preamble to replica: %s", connGetLastError(conn)); freeClient(replica); return; @@ -1985,12 +2025,14 @@ void readSyncBulkPayload(connection *conn) { "Failed trying to load the MASTER synchronization " "DB from disk"); cancelReplicationHandshake(mi); + if (allPersistenceDisabled()) unlink(g_pserver->rdb_filename); /* Note that there's no point in restarting the AOF on sync failure, it'll be restarted when sync succeeds or replica promoted. */ return; } /* Cleanup. */ + if (allPersistenceDisabled()) unlink(server.rdb_filename); if (fUpdate) unlink(mi->repl_transfer_tmpfile); zfree(mi->repl_transfer_tmpfile); @@ -3684,6 +3726,11 @@ void replicationCron(void) { propagateMasterStaleKeys(); + + /* Remove the RDB file used for replication if Redis is not running + * with any persistence. */ + removeRDBUsedToSyncReplicas(); + /* Refresh the number of slaves with lag <= min-slaves-max-lag. */ refreshGoodSlavesCount(); replication_cron_loops++; /* Incremented with frequency 1 HZ. */ diff --git a/src/server.cpp b/src/server.cpp index 874d391c6..30f4f4099 100644 --- a/src/server.cpp +++ b/src/server.cpp @@ -1483,12 +1483,20 @@ void updateDictResizePolicy(void) { dictDisableResize(); } +/* Return true if there are no active children processes doing RDB saving, + * AOF rewriting, or some side process spawned by a loaded module. */ int hasActiveChildProcess() { return g_pserver->rdb_child_pid != -1 || g_pserver->aof_child_pid != -1 || g_pserver->module_child_pid != -1; } +/* Return true if this instance has persistence completely turned off: + * both RDB and AOF are disabled. */ +int allPersistenceDisabled(void) { + return server.saveparamslen == 0 && server.aof_state == AOF_OFF; +} + /* ======================= Cron: called every 100 ms ======================== */ /* Add a sample to the operations per second array of samples. */ diff --git a/src/server.h b/src/server.h index b8a11e888..b9bd9b7c4 100644 --- a/src/server.h +++ b/src/server.h @@ -2357,6 +2357,7 @@ void loadingProgress(off_t pos); void stopLoading(int success); void startSaving(int rdbflags); void stopSaving(int success); +int allPersistenceDisabled(void); #define DISK_ERROR_TYPE_AOF 1 /* Don't accept writes: AOF errors. */ #define DISK_ERROR_TYPE_RDB 2 /* Don't accept writes: RDB errors. */ From f084ee6f173a1a06fadbe0635b6ea28c580966ec Mon Sep 17 00:00:00 2001 From: John Sully Date: Tue, 14 Apr 2020 17:33:54 -0400 Subject: [PATCH 206/225] Fix merge conflicts from cherry-pick Former-commit-id: 57956b3fd2d05581c976f58d07e245896d3a515b --- src/replication.cpp | 13 +++++++------ src/server.cpp | 4 ++-- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/src/replication.cpp b/src/replication.cpp index 3d0cafc56..29ec9e26e 100644 --- a/src/replication.cpp +++ b/src/replication.cpp @@ -1147,15 +1147,16 @@ void putSlaveOnline(client *replica) { * to take RDB files around, this violates certain policies in certain * environments. */ void removeRDBUsedToSyncReplicas(void) { + serverAssert(GlobalLocksAcquired()); if (allPersistenceDisabled() && RDBGeneratedByReplication) { client *slave; listNode *ln; listIter li; int delrdb = 1; - listRewind(server.slaves,&li); + listRewind(g_pserver->slaves,&li); while((ln = listNext(&li))) { - slave = ln->value; + slave = (client*)ln->value; if (slave->replstate == SLAVE_STATE_WAIT_BGSAVE_START || slave->replstate == SLAVE_STATE_WAIT_BGSAVE_END || slave->replstate == SLAVE_STATE_SEND_BULK) @@ -1166,7 +1167,7 @@ void removeRDBUsedToSyncReplicas(void) { } if (delrdb) { RDBGeneratedByReplication = 0; - unlink(server.rdb_filename); + unlink(g_pserver->rdb_filename); } } } @@ -1237,7 +1238,7 @@ void rdbPipeWriteHandlerConnRemoved(struct connection *conn) { /* if there are no more writes for now for this conn, or write error: */ if (g_pserver->rdb_pipe_numconns_writing == 0) { if (aeCreateFileEvent(serverTL->el, g_pserver->rdb_pipe_read, AE_READABLE, rdbPipeReadHandler,NULL) == AE_ERR) { - serverPanic("Unrecoverable error creating server.rdb_pipe_read file event."); + serverPanic("Unrecoverable error creating g_pserver->rdb_pipe_read file event."); } } } @@ -2032,7 +2033,7 @@ void readSyncBulkPayload(connection *conn) { } /* Cleanup. */ - if (allPersistenceDisabled()) unlink(server.rdb_filename); + if (allPersistenceDisabled()) unlink(g_pserver->rdb_filename); if (fUpdate) unlink(mi->repl_transfer_tmpfile); zfree(mi->repl_transfer_tmpfile); @@ -2626,7 +2627,7 @@ void syncWithMaster(connection *conn) { if (psync_result == PSYNC_CONTINUE) { serverLog(LL_NOTICE, "MASTER <-> REPLICA sync: Master accepted a Partial Resynchronization."); - if (server.supervised_mode == SUPERVISED_SYSTEMD) { + if (cserver.supervised_mode == SUPERVISED_SYSTEMD) { redisCommunicateSystemd("STATUS=MASTER <-> REPLICA sync: Partial Resynchronization accepted. Ready to accept connections.\n"); redisCommunicateSystemd("READY=1\n"); } diff --git a/src/server.cpp b/src/server.cpp index 30f4f4099..5f0a72d69 100644 --- a/src/server.cpp +++ b/src/server.cpp @@ -1494,7 +1494,7 @@ int hasActiveChildProcess() { /* Return true if this instance has persistence completely turned off: * both RDB and AOF are disabled. */ int allPersistenceDisabled(void) { - return server.saveparamslen == 0 && server.aof_state == AOF_OFF; + return g_pserver->saveparamslen == 0 && g_pserver->aof_state == AOF_OFF; } /* ======================= Cron: called every 100 ms ======================== */ @@ -3488,7 +3488,7 @@ void call(client *c, int flags) { if (flags & CMD_CALL_PROPAGATE) { bool multi_emitted = false; - /* Wrap the commands in server.also_propagate array, + /* Wrap the commands in g_pserver->also_propagate array, * but don't wrap it if we are already in MULIT context, * in case the nested MULIT/EXEC. * From 10a7bfe3a2f5189e3546c4e638a1a8f82543d824 Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Thu, 6 Feb 2020 09:41:45 +0200 Subject: [PATCH 207/225] reduce repeated calls to use_diskless_load this function possibly iterates on the module list Former-commit-id: 99762dac23e26a217fcd66531ee4ca9b4d13a7ac --- src/replication.cpp | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/replication.cpp b/src/replication.cpp index 29ec9e26e..94d4a7fbe 100644 --- a/src/replication.cpp +++ b/src/replication.cpp @@ -1719,7 +1719,7 @@ void disklessLoadRestoreBackups(redisDb *backup, int restore, int empty_db_flags void readSyncBulkPayload(connection *conn) { char buf[4096]; ssize_t nread, readlen, nwritten; - int use_diskless_load; + int use_diskless_load = useDisklessLoad(); redisDb *diskless_load_backup = NULL; rdbSaveInfo rsi = RDB_SAVE_INFO_INIT; int empty_db_flags = g_pserver->repl_slave_lazy_flush ? EMPTYDB_ASYNC : @@ -1782,19 +1782,18 @@ void readSyncBulkPayload(connection *conn) { mi->repl_transfer_size = 0; serverLog(LL_NOTICE, "MASTER <-> REPLICA sync: receiving streamed RDB from master with EOF %s", - useDisklessLoad()? "to parser":"to disk"); + use_diskless_load? "to parser":"to disk"); } else { usemark = 0; mi->repl_transfer_size = strtol(buf+1,NULL,10); serverLog(LL_NOTICE, "MASTER <-> REPLICA sync: receiving %lld bytes from master %s", (long long) mi->repl_transfer_size, - useDisklessLoad() ? "to parser" : "to disk"); + use_diskless_load? "to parser":"to disk"); } return; } - use_diskless_load = useDisklessLoad(); if (!use_diskless_load) { /* Read the data from the socket, store it to a file and search * for the EOF. */ From 0fac746f0ea9e7041f4e1cc684a63add5009998e Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 24 Mar 2020 11:02:40 +0100 Subject: [PATCH 208/225] PSYNC2: meaningful offset implemented. A very commonly signaled operational problem with Redis master-replicas sets is that, once the master becomes unavailable for some reason, especially because of network problems, many times it wont be able to perform a partial resynchronization with the new master, once it rejoins the partition, for the following reason: 1. The master becomes isolated, however it keeps sending PINGs to the replicas. Such PINGs will never be received since the link connection is actually already severed. 2. On the other side, one of the replicas will turn into the new master, setting its secondary replication ID offset to the one of the last command received from the old master: this offset will not include the PINGs sent by the master once the link was already disconnected. 3. When the master rejoins the partion and is turned into a replica, its offset will be too advanced because of the PINGs, so a PSYNC will fail, and a full synchronization will be required. Related to issue #7002 and other discussion we had in the past around this problem. Former-commit-id: 5d6e8fe3e3e43162f0c57f580b6e8432274fca30 --- src/replication.cpp | 42 +++++++++++++++++++++++++++++++++++++++++- src/server.cpp | 4 ++++ src/server.h | 1 + 3 files changed, 46 insertions(+), 1 deletion(-) diff --git a/src/replication.cpp b/src/replication.cpp index 94d4a7fbe..a7a95ed01 100644 --- a/src/replication.cpp +++ b/src/replication.cpp @@ -219,6 +219,7 @@ void feedReplicationBacklog(const void *ptr, size_t len) { const unsigned char *p = (const unsigned char*)ptr; g_pserver->master_repl_offset += len; + g_pserver->master_repl_meaningful_offset = g_pserver->master_repl_offset; /* This is a circular buffer, so write as much data we can at every * iteration and rewind the "idx" index if we reach the limit. */ @@ -2066,6 +2067,7 @@ void readSyncBulkPayload(connection *conn) { * we are starting a new history. */ memcpy(g_pserver->replid,mi->master->replid,sizeof(g_pserver->replid)); g_pserver->master_repl_offset = mi->master->reploff; + g_pserver->master_repl_meaningful_offset = mi->master->reploff; } clearReplicationId2(); @@ -3159,12 +3161,43 @@ void replicationCacheMaster(redisMaster *mi, client *c) { * current offset if no data was lost during the failover. So we use our * current replication ID and offset in order to synthesize a cached master. */ void replicationCacheMasterUsingMyself(redisMaster *mi) { + serverLog(LL_NOTICE, + "Before turning into a replica, using my own master parameters " + "to synthesize a cached master: I may be able to synchronize with " + "the new master with just a partial transfer."); + if (mi->cached_master != nullptr) { // This can happen on first load of the RDB, the master we created in config load is stale freeClient(mi->cached_master); } + /* This will be used to populate the field server.master->reploff + * by replicationCreateMasterClient(). We'll later set the created + * master as server.cached_master, so the replica will use such + * offset for PSYNC. */ + mi->master_initial_offset = g_pserver->master_repl_offset; + + /* However if the "meaningful" offset, that is the offset without + * the final PINGs in the stream, is different, use this instead: + * often when the master is no longer reachable, replicas will never + * receive the PINGs, however the master will end with an incremented + * offset because of the PINGs and will not be able to incrementally + * PSYNC with the new master. */ + if (g_pserver->master_repl_offset > g_pserver->master_repl_meaningful_offset) { + long long delta = g_pserver->master_repl_offset - + g_pserver->master_repl_meaningful_offset; + serverLog(LL_NOTICE, + "Using the meaningful offset %lld instead of %lld to exclude " + "the final PINGs (%lld bytes difference)", + g_pserver->master_repl_meaningful_offset, + g_pserver->master_repl_offset, + delta); + mi->master_initial_offset = g_pserver->master_repl_meaningful_offset; + g_pserver->repl_backlog_histlen -= delta; + if (g_pserver->repl_backlog_histlen < 0) g_pserver->repl_backlog_histlen = 0; + } + /* The master client we create can be set to any DBID, because * the new master will start its replication stream with SELECT. */ mi->master_initial_offset = g_pserver->master_repl_offset; @@ -3178,7 +3211,6 @@ void replicationCacheMasterUsingMyself(redisMaster *mi) { unlinkClient(mi->master); mi->cached_master = mi->master; mi->master = NULL; - serverLog(LL_NOTICE,"Before turning into a replica, using my master parameters to synthesize a cached master: I may be able to synchronize with the new master with just a partial transfer."); } /* Free a cached master, called when there are no longer the conditions for @@ -3583,10 +3615,18 @@ void replicationCron(void) { clientsArePaused(); if (!manual_failover_in_progress) { + long long before_ping = g_pserver->master_repl_meaningful_offset; ping_argv[0] = createStringObject("PING",4); replicationFeedSlaves(g_pserver->slaves, g_pserver->replicaseldb, ping_argv, 1); decrRefCount(ping_argv[0]); + /* The server.master_repl_meaningful_offset variable represents + * the offset of the replication stream without the pending PINGs. + * This is useful to set the right replication offset for PSYNC + * when the master is turned into a replica. Otherwise pending + * PINGs may not allow it to perform an incremental sync with the + * new master. */ + g_pserver->master_repl_meaningful_offset = before_ping; } } diff --git a/src/server.cpp b/src/server.cpp index 5f0a72d69..000bdd76f 100644 --- a/src/server.cpp +++ b/src/server.cpp @@ -2475,6 +2475,7 @@ void initServerConfig(void) { g_pserver->enable_multimaster = CONFIG_DEFAULT_ENABLE_MULTIMASTER; g_pserver->repl_syncio_timeout = CONFIG_REPL_SYNCIO_TIMEOUT; g_pserver->master_repl_offset = 0; + g_pserver->master_repl_meaningful_offset = 0; /* Replication partial resync backlog */ g_pserver->repl_backlog = NULL; @@ -4626,6 +4627,7 @@ sds genRedisInfoString(const char *section) { "master_replid:%s\r\n" "master_replid2:%s\r\n" "master_repl_offset:%lld\r\n" + "master_repl_meaningful_offset:%lld\r\n" "second_repl_offset:%lld\r\n" "repl_backlog_active:%d\r\n" "repl_backlog_size:%lld\r\n" @@ -4634,6 +4636,7 @@ sds genRedisInfoString(const char *section) { g_pserver->replid, g_pserver->replid2, g_pserver->master_repl_offset, + g_pserver->master_repl_meaningful_offset, g_pserver->second_replid_offset, g_pserver->repl_backlog != NULL, g_pserver->repl_backlog_size, @@ -5027,6 +5030,7 @@ void loadDataFromDisk(void) { { memcpy(g_pserver->replid,rsi.repl_id,sizeof(g_pserver->replid)); g_pserver->master_repl_offset = rsi.repl_offset; + g_pserver->master_repl_meaningful_offset = rsi.repl_offset; listIter li; listNode *ln; diff --git a/src/server.h b/src/server.h index b9bd9b7c4..80e6f1f1b 100644 --- a/src/server.h +++ b/src/server.h @@ -1792,6 +1792,7 @@ struct redisServer { char replid[CONFIG_RUN_ID_SIZE+1]; /* My current replication ID. */ char replid2[CONFIG_RUN_ID_SIZE+1]; /* replid inherited from master*/ long long master_repl_offset; /* My current replication offset */ + long long master_repl_meaningful_offset; /* Offset minus latest PINGs. */ long long second_replid_offset; /* Accept offsets up to this for replid2. */ int replicaseldb; /* Last SELECTed DB in replication output */ int repl_ping_slave_period; /* Master pings the replica every N seconds */ From eafcca9f29b8325b67183aa7ce65516a4d998c09 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 13 Jan 2020 13:16:13 +0100 Subject: [PATCH 209/225] Jump to right label on AOF parsing error. Related to #6054. Former-commit-id: c7159d58f118e840a3b9f72160d1abf3e8a86b3d --- src/aof.cpp | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/aof.cpp b/src/aof.cpp index 291685e5c..da1ae3c14 100644 --- a/src/aof.cpp +++ b/src/aof.cpp @@ -815,12 +815,14 @@ int loadAppendOnlyFile(char *filename) { for (j = 0; j < argc; j++) { /* Parse the argument len. */ - if (fgets(buf,sizeof(buf),fp) == NULL || - buf[0] != '$') - { + char *readres = fgets(buf,sizeof(buf),fp); + if (readres == NULL || buf[0] != '$') { fakeClient->argc = j; /* Free up to j-1. */ freeFakeClientArgv(fakeClient); - goto readerr; + if (readres == NULL) + goto readerr; + else + goto fmterr; } len = strtol(buf+1,NULL,10); From 8675bd3bdee87c9614b1eda9b9564d456626586b Mon Sep 17 00:00:00 2001 From: John Sully Date: Tue, 14 Apr 2020 22:37:26 -0400 Subject: [PATCH 210/225] Fix merge issues and move timeout to C++ Former-commit-id: 1005a725d498e3c9f8c708d3c8b013a402149bd8 --- src/{timeout.c => timeout.cpp} | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) rename src/{timeout.c => timeout.cpp} (97%) diff --git a/src/timeout.c b/src/timeout.cpp similarity index 97% rename from src/timeout.c rename to src/timeout.cpp index 6328bcd5d..4d114a25e 100644 --- a/src/timeout.c +++ b/src/timeout.cpp @@ -28,6 +28,7 @@ #include "server.h" #include "cluster.h" +#include /* ========================== Clients timeouts ============================= */ @@ -163,10 +164,10 @@ void clientsHandleTimeout(void) { /* This function is called in beforeSleep() in order to unblock clients * that are waiting in blocking operations with a timeout set. */ void handleBlockedClientsTimeout(void) { - if (raxSize(server.clients_timeout_table) == 0) return; + if (raxSize(g_pserver->clients_timeout_table) == 0) return; uint64_t now = mstime(); raxIterator ri; - raxStart(&ri,server.clients_timeout_table); + raxStart(&ri,g_pserver->clients_timeout_table); raxSeek(&ri,"^",NULL,0); while(raxNext(&ri)) { @@ -175,10 +176,11 @@ void handleBlockedClientsTimeout(void) { if (timeout >= now) break; /* All the timeouts are in the future. */ client *c = lookupClientByID(id); if (c) { + std::unique_lock lock(c->lock); c->flags &= ~CLIENT_IN_TO_TABLE; checkBlockedClientTimeout(c,now); } - raxRemove(server.clients_timeout_table,ri.key,ri.key_len,NULL); + raxRemove(g_pserver->clients_timeout_table,ri.key,ri.key_len,NULL); raxSeek(&ri,"^",NULL,0); } } From 4ee29a3b2500dfa7e0b1b1c0b617c6bbd2196988 Mon Sep 17 00:00:00 2001 From: John Sully Date: Wed, 15 Apr 2020 16:34:07 -0400 Subject: [PATCH 211/225] Fix incorrect cluster slot tracking (regression from merge) Former-commit-id: 4705f29e2f62d90c374e072319c8cd486d32f807 --- src/db.cpp | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/src/db.cpp b/src/db.cpp index db95113d8..666812982 100644 --- a/src/db.cpp +++ b/src/db.cpp @@ -204,8 +204,9 @@ int dbAddCore(redisDb *db, robj *key, robj *val) { if (retval == DICT_OK) { if (val->type == OBJ_LIST || - val->type == OBJ_ZSET) - signalKeyAsReady(db, key); + val->type == OBJ_ZSET || + val->type == OBJ_STREAM) + signalKeyAsReady(db, key); if (g_pserver->cluster_enabled) slotToKeyAdd(key); } else @@ -224,11 +225,6 @@ void dbAdd(redisDb *db, robj *key, robj *val) { int retval = dbAddCore(db, key, val); serverAssertWithInfo(NULL,key,retval == DICT_OK); - if (val->type == OBJ_LIST || - val->type == OBJ_ZSET || - val->type == OBJ_STREAM) - signalKeyAsReady(db, key); - if (g_pserver->cluster_enabled) slotToKeyAdd(key); } void dbOverwriteCore(redisDb *db, dictEntry *de, robj *key, robj *val, bool fUpdateMvcc, bool fRemoveExpire) @@ -1880,6 +1876,8 @@ int *xreadGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numkeys) * while rehashing the cluster and in other conditions when we need to * understand if we have keys for a given hash slot. */ void slotToKeyUpdateKey(robj *key, int add) { + serverAssert(GlobalLocksAcquired()); + size_t keylen = sdslen(szFromObj(key)); unsigned int hashslot = keyHashSlot(szFromObj(key),keylen); unsigned char buf[64]; @@ -1890,11 +1888,13 @@ void slotToKeyUpdateKey(robj *key, int add) { indexed[0] = (hashslot >> 8) & 0xff; indexed[1] = hashslot & 0xff; memcpy(indexed+2,ptrFromObj(key),keylen); + int fModified = false; if (add) { - raxInsert(g_pserver->cluster->slots_to_keys,indexed,keylen+2,NULL,NULL); + fModified = raxInsert(g_pserver->cluster->slots_to_keys,indexed,keylen+2,NULL,NULL); } else { - raxRemove(g_pserver->cluster->slots_to_keys,indexed,keylen+2,NULL); + fModified = raxRemove(g_pserver->cluster->slots_to_keys,indexed,keylen+2,NULL); } + serverAssert(fModified); if (indexed != buf) zfree(indexed); } @@ -1907,6 +1907,8 @@ void slotToKeyDel(robj *key) { } void slotToKeyFlush(void) { + serverAssert(GlobalLocksAcquired()); + raxFree(g_pserver->cluster->slots_to_keys); g_pserver->cluster->slots_to_keys = raxNew(); memset(g_pserver->cluster->slots_keys_count,0, @@ -1936,6 +1938,8 @@ unsigned int getKeysInSlot(unsigned int hashslot, robj **keys, unsigned int coun /* Remove all the keys in the specified hash slot. * The number of removed items is returned. */ unsigned int delKeysInSlot(unsigned int hashslot) { + serverAssert(GlobalLocksAcquired()); + raxIterator iter; int j = 0; unsigned char indexed[2]; @@ -1947,8 +1951,10 @@ unsigned int delKeysInSlot(unsigned int hashslot) { raxSeek(&iter,">=",indexed,2); raxNext(&iter); + auto count = g_pserver->cluster->slots_keys_count[hashslot]; robj *key = createStringObject((char*)iter.key+2,iter.key_len-2); dbDelete(&g_pserver->db[0],key); + serverAssert(count > g_pserver->cluster->slots_keys_count[hashslot]); // we should have deleted something or we will be in an infinite loop decrRefCount(key); j++; } From fc66f768e044e40b0b531f324c8c472028c96c48 Mon Sep 17 00:00:00 2001 From: John Sully Date: Wed, 15 Apr 2020 16:34:38 -0400 Subject: [PATCH 212/225] Fix TSAN race Former-commit-id: f00d28fdabe858bd621a1bd98e40493aca5aba1e --- src/object.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/object.cpp b/src/object.cpp index 8ef69a0eb..77fc1c732 100644 --- a/src/object.cpp +++ b/src/object.cpp @@ -32,6 +32,7 @@ #include "cron.h" #include #include +#include #ifdef __CYGWIN__ #define strtold(a,b) ((long double)strtod((a),(b))) @@ -1033,6 +1034,8 @@ struct redisMemOverhead *getMemoryOverheadData(void) { while((ln = listNext(&li))) { size_t mem_curr = 0; client *c = (client*)listNodeValue(ln); + std::unique_lock ul(c->lock); + int type = getClientType(c); mem_curr += getClientOutputBufferMemoryUsage(c); mem_curr += sdsAllocSize(c->querybuf); From fb991c8e954eb34e8bcebb31399a6acf53ee310a Mon Sep 17 00:00:00 2001 From: John Sully Date: Wed, 15 Apr 2020 16:35:24 -0400 Subject: [PATCH 213/225] Log stdout and stderr for cluster tests Former-commit-id: 06143c2e8cab5c201ce41b85fcac70a36f2626c1 --- tests/instances.tcl | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/tests/instances.tcl b/tests/instances.tcl index c3102890c..c02a6697f 100644 --- a/tests/instances.tcl +++ b/tests/instances.tcl @@ -37,7 +37,7 @@ if {[catch {cd tmp}]} { # Execute the specified instance of the server specified by 'type', using # the provided configuration file. Returns the PID of the process. -proc exec_instance {type cfgfile} { +proc exec_instance {type cfgfile dir} { if {$type eq "redis"} { set prgname keydb-server } elseif {$type eq "sentinel"} { @@ -46,10 +46,13 @@ proc exec_instance {type cfgfile} { error "Unknown instance type." } + set stdout [format "%s/%s" $dir "stdout"] + set stderr [format "%s/%s" $dir "stderr"] + if {$::valgrind} { set pid [exec valgrind --track-origins=yes --suppressions=../../../src/valgrind.sup --show-reachable=no --show-possibly-lost=no --leak-check=full ../../../src/${prgname} $cfgfile &] } else { - set pid [exec ../../../src/${prgname} $cfgfile &] + set pid [exec ../../../src/${prgname} $cfgfile > $stdout 2> $stderr &] } return $pid } @@ -92,7 +95,7 @@ proc spawn_instance {type base_port count {conf {}}} { close $cfg # Finally exec it and remember the pid for later cleanup. - set pid [exec_instance $type $cfgfile] + set pid [exec_instance $type $cfgfile $dirname] lappend ::pids $pid # Check availability @@ -502,7 +505,7 @@ proc restart_instance {type id} { # Execute the instance with its old setup and append the new pid # file for cleanup. - set pid [exec_instance $type $cfgfile] + set pid [exec_instance $type $cfgfile $dirname] set_instance_attrib $type $id pid $pid lappend ::pids $pid From 9b25f954d0fe239f5c8577ff11f2cacfb29c271c Mon Sep 17 00:00:00 2001 From: John Sully Date: Wed, 15 Apr 2020 16:44:06 -0400 Subject: [PATCH 214/225] rename to KeyDB (merge) Former-commit-id: 2926ac494e76c641c19826565db8224ae533d8a3 --- tests/cluster/tests/14-consistency-check.tcl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/cluster/tests/14-consistency-check.tcl b/tests/cluster/tests/14-consistency-check.tcl index a43725ebc..bea039866 100644 --- a/tests/cluster/tests/14-consistency-check.tcl +++ b/tests/cluster/tests/14-consistency-check.tcl @@ -64,7 +64,7 @@ proc test_slave_load_expired_keys {aof} { kill_instance redis $replica_id set master_port [get_instance_attrib redis $master_id port] - exec ../../../src/redis-cli -h 127.0.0.1 -p $master_port debug sleep [expr $data_ttl+3] > /dev/null & + exec ../../../src/keydb-cli -h 127.0.0.1 -p $master_port debug sleep [expr $data_ttl+3] > /dev/null & while {[clock seconds] <= $end_time} { #wait for $data_ttl seconds From 24223ed818ec6e5ef46945793aed2b08f0a4acb3 Mon Sep 17 00:00:00 2001 From: John Sully Date: Wed, 15 Apr 2020 20:52:25 -0400 Subject: [PATCH 215/225] Multithreading reliability, force single thread for test relying on internal behavior Former-commit-id: 033761c5f97fc1d1823a031b34467ac1df5588f3 --- tests/integration/aof.tcl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/integration/aof.tcl b/tests/integration/aof.tcl index 6b1b64b85..9b605c266 100644 --- a/tests/integration/aof.tcl +++ b/tests/integration/aof.tcl @@ -258,7 +258,9 @@ tags {"aof"} { } } - start_server {overrides {appendonly {yes} appendfilename {appendonly.aof} appendfsync always}} { + # Because of how this test works its inherently unreliable with multithreading, so force threads 1 + # No real client should rely on this undocumented behavior + start_server {overrides {appendonly {yes} appendfilename {appendonly.aof} appendfsync always server-threads 1}} { test {AOF fsync always barrier issue} { set rd [redis_deferring_client] # Set a sleep when aof is flushed, so that we have a chance to look From 4f2f2affed28c3d394f72e4690cdc5b4a1d7928e Mon Sep 17 00:00:00 2001 From: John Sully Date: Wed, 15 Apr 2020 22:24:12 -0400 Subject: [PATCH 216/225] ASAN races in leak checker Former-commit-id: 9dfa074cd6d5ed9a87036e582861dfc386b56d5e --- src/fastlock.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/fastlock.cpp b/src/fastlock.cpp index 861bfb7ca..5fe5c59db 100644 --- a/src/fastlock.cpp +++ b/src/fastlock.cpp @@ -222,7 +222,8 @@ public: auto itr = m_mapwait.find(pidCheck); if (itr == m_mapwait.end()) break; - pidCheck = itr->second->m_pidOwner; + + __atomic_load(&itr->second->m_pidOwner, &pidCheck, __ATOMIC_RELAXED); if (pidCheck == thispid) { // Deadlock detected, printout some debugging info and crash @@ -233,7 +234,7 @@ public: { auto itr = m_mapwait.find(pidCheck); serverLog(3 /* LL_WARNING */, "\t%d: (%p) %s", pidCheck, itr->second, itr->second->szName); - pidCheck = itr->second->m_pidOwner; + __atomic_load(&itr->second->m_pidOwner, &pidCheck, __ATOMIC_RELAXED); if (pidCheck == thispid) break; } From bbb08af2941a5e2f7dab38434f6bf82de8b40a4d Mon Sep 17 00:00:00 2001 From: John Sully Date: Wed, 15 Apr 2020 22:25:17 -0400 Subject: [PATCH 217/225] Convert variables accessed outside lock to atomics Former-commit-id: b0796ff5fd7e069a2fadbfd968f7bbb2020edd2d --- src/server.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/server.h b/src/server.h index 24fa8a6f1..10941adfc 100644 --- a/src/server.h +++ b/src/server.h @@ -1631,7 +1631,7 @@ struct redisServer { struct redisServerThreadVars rgthreadvar[MAX_EVENT_LOOPS]; std::atomic lruclock; /* Clock for LRU eviction */ - int shutdown_asap; /* SHUTDOWN needed ASAP */ + std::atomic shutdown_asap; /* SHUTDOWN needed ASAP */ int activerehashing; /* Incremental rehash in serverCron() */ int active_defrag_running; /* Active defragmentation running (holds current scan aggressiveness) */ int cronloops; /* Number of times the cron function run */ @@ -1666,7 +1666,7 @@ struct redisServer { std::atomic next_client_id; /* Next client unique ID. Incremental. */ int protected_mode; /* Don't accept external connections. */ /* RDB / AOF loading information */ - int loading; /* We are loading data from disk if true */ + std::atomic loading; /* We are loading data from disk if true */ off_t loading_total_bytes; off_t loading_loaded_bytes; time_t loading_start_time; From 19550c4819a7d335acd78d43bcd9ca0850cafb52 Mon Sep 17 00:00:00 2001 From: John Sully Date: Wed, 15 Apr 2020 22:26:00 -0400 Subject: [PATCH 218/225] Fix race in sendBulk Former-commit-id: 5fd07e08894482e1a55f18ece9c52ff5379b82ec --- src/replication.cpp | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/replication.cpp b/src/replication.cpp index bc97773b6..ded3fa01f 100644 --- a/src/replication.cpp +++ b/src/replication.cpp @@ -1228,6 +1228,7 @@ void sendBulkToSlave(connection *conn) { serverAssert(FCorrectThread(replica)); char buf[PROTO_IOBUF_LEN]; ssize_t nwritten, buflen; + std::unique_lock ul(replica->lock); /* Before sending the RDB file, we send the preamble as configured by the * replication process. Currently the preamble is just the bulk count of @@ -3109,6 +3110,10 @@ void roleCommand(client *c) { while ((ln = listNext(&li))) { redisMaster *mi = (redisMaster*)listNodeValue(ln); + std::unique_lock lock; + if (mi->master != nullptr) + lock = std::unique_lock(mi->master->lock); + const char *slavestate = NULL; addReplyArrayLen(c,5); if (g_pserver->fActiveReplica) @@ -3952,6 +3957,13 @@ struct RemoteMasterState { uint64_t mvcc = 0; client *cFake = nullptr; + + ~RemoteMasterState() + { + aeAcquireLock(); + freeClient(cFake); + aeReleaseLock(); + } }; static std::unordered_map g_mapremote; From 3e2ba10d005ac80736a4005f31de9d85a9c9a9ac Mon Sep 17 00:00:00 2001 From: John Sully Date: Wed, 15 Apr 2020 22:27:04 -0400 Subject: [PATCH 219/225] Run all KeyDB instances in testmode during tests Former-commit-id: cd306f1d23f4fbb900433edbf55d89099bbf903c --- tests/support/server.tcl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/support/server.tcl b/tests/support/server.tcl index 6b5df039c..69326bb20 100644 --- a/tests/support/server.tcl +++ b/tests/support/server.tcl @@ -230,6 +230,9 @@ proc start_server {options {code undefined}} { set unixsocket [file normalize [format "%s/%s" [dict get $config "dir"] "socket"]] dict set config "unixsocket" $unixsocket + #Ensure all tests validate multithreading + dict set config "testmode" "yes" + # apply overrides from global space and arguments foreach {directive arguments} [concat $::global_overrides $overrides] { dict set config $directive $arguments From 470a895585ac3d9a40edec10b69fb220605361d9 Mon Sep 17 00:00:00 2001 From: John Sully Date: Wed, 15 Apr 2020 22:27:45 -0400 Subject: [PATCH 220/225] Quiet test only ASAN fd race Former-commit-id: d4939c838b58eab2fb3b631267045cff9d3caff1 --- src/server.cpp | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/server.cpp b/src/server.cpp index ea5b2ef4a..f080adf2b 100644 --- a/src/server.cpp +++ b/src/server.cpp @@ -2565,7 +2565,17 @@ int restartServer(int flags, mstime_t delay) { for (j = 3; j < (int)g_pserver->maxclients + 1024; j++) { /* Test the descriptor validity before closing it, otherwise * Valgrind issues a warning on close(). */ - if (fcntl(j,F_GETFD) != -1) close(j); + if (fcntl(j,F_GETFD) != -1) + { + /* This user to just close() here, but sanitizers detected that as an FD race. + The race doesn't matter since we're about to call exec() however we want + to cut down on noise, so instead we ask the kernel to close when we call + exec(), and only do it ourselves if that fails. */ + if (fcntl(j, F_SETFD, FD_CLOEXEC) == -1) + { + close(j); // failed to set close on exec, close here + } + } } /* Execute the server with the original command line. */ @@ -4374,7 +4384,7 @@ sds genRedisInfoString(const char *section) { "aof_last_cow_size:%zu\r\n" "module_fork_in_progress:%d\r\n" "module_fork_last_cow_size:%zu\r\n", - g_pserver->loading, + g_pserver->loading.load(std::memory_order_relaxed), g_pserver->dirty, g_pserver->rdb_child_pid != -1, (intmax_t)g_pserver->lastsave, From 34131bf287ba6a19b58ae54a656a45a272818dcf Mon Sep 17 00:00:00 2001 From: John Sully Date: Wed, 15 Apr 2020 22:30:19 -0400 Subject: [PATCH 221/225] During AOF reload we can erroneously read incorrect aof_state values, so this variable must be read with the global lock acquired Former-commit-id: 6ff9d23fd4541a011d754209d9fda3ef3af4a7f9 --- src/networking.cpp | 7 ++++--- src/server.cpp | 6 ++++-- src/server.h | 2 +- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/src/networking.cpp b/src/networking.cpp index c8473b89a..1cb9eade6 100644 --- a/src/networking.cpp +++ b/src/networking.cpp @@ -1856,7 +1856,7 @@ void ProcessPendingAsyncWrites() * we can just write the replies to the client output buffer without any * need to use a syscall in order to install the writable event handler, * get it called, and so forth. */ -int handleClientsWithPendingWrites(int iel) { +int handleClientsWithPendingWrites(int iel, int aof_state) { std::unique_lock lockf(g_pserver->rgthreadvar[iel].lockPendingWrite); auto &vec = g_pserver->rgthreadvar[iel].clients_pending_write; int processed = (int)vec.size(); @@ -1868,7 +1868,7 @@ int handleClientsWithPendingWrites(int iel) { * so that in the middle of receiving the query, and serving it * to the client, we'll call beforeSleep() that will do the * actual fsync of AOF to disk. AE_BARRIER ensures that. */ - if (g_pserver->aof_state == AOF_ON && + if (aof_state == AOF_ON && g_pserver->aof_fsync == AOF_FSYNC_ALWAYS) { ae_flags |= AE_BARRIER; @@ -3359,6 +3359,7 @@ int processEventsWhileBlocked(int iel) { } + int aof_state = g_pserver->aof_state; aeReleaseLock(); serverAssertDebug(!GlobalLocksAcquired()); try @@ -3366,7 +3367,7 @@ int processEventsWhileBlocked(int iel) { while (iterations--) { int events = 0; events += aeProcessEvents(g_pserver->rgthreadvar[iel].el, AE_FILE_EVENTS|AE_DONT_WAIT); - events += handleClientsWithPendingWrites(iel); + events += handleClientsWithPendingWrites(iel, aof_state); if (!events) break; count += events; } diff --git a/src/server.cpp b/src/server.cpp index f080adf2b..1ebf71b1e 100644 --- a/src/server.cpp +++ b/src/server.cpp @@ -2191,8 +2191,9 @@ void beforeSleep(struct aeEventLoop *eventLoop) { flushAppendOnlyFile(0); /* Handle writes with pending output buffers. */ + int aof_state = g_pserver->aof_state; aeReleaseLock(); - handleClientsWithPendingWrites(IDX_EVENT_LOOP_MAIN); + handleClientsWithPendingWrites(IDX_EVENT_LOOP_MAIN, aof_state); aeAcquireLock(); /* Close clients that need to be closed asynchronous */ @@ -2217,10 +2218,11 @@ void beforeSleepLite(struct aeEventLoop *eventLoop) /* Check if there are clients unblocked by modules that implement * blocking commands. */ if (moduleCount()) moduleHandleBlockedClients(ielFromEventLoop(eventLoop)); + int aof_state = g_pserver->aof_state; aeReleaseLock(); /* Handle writes with pending output buffers. */ - handleClientsWithPendingWrites(iel); + handleClientsWithPendingWrites(iel, aof_state); aeAcquireLock(); /* Close clients that need to be closed asynchronous */ diff --git a/src/server.h b/src/server.h index 10941adfc..3a08b4d6a 100644 --- a/src/server.h +++ b/src/server.h @@ -2203,7 +2203,7 @@ void pauseClients(mstime_t duration); int clientsArePaused(void); void unpauseClientsIfNecessary(); int processEventsWhileBlocked(int iel); -int handleClientsWithPendingWrites(int iel); +int handleClientsWithPendingWrites(int iel, int aof_state); int clientHasPendingReplies(client *c); void unlinkClient(client *c); int writeToClient(client *c, int handler_installed); From a7d88c1287afc07fd1a11ad3cf69e58de40fdb87 Mon Sep 17 00:00:00 2001 From: John Sully Date: Wed, 15 Apr 2020 22:36:16 -0400 Subject: [PATCH 222/225] Add CI dependency Former-commit-id: 3137d4f35c5ae5b24a8956fadeec2d93c7048165 --- .github/workflows/ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cc4991606..0d46f0b03 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,7 +8,9 @@ jobs: steps: - uses: actions/checkout@v1 - name: make - run: make + run: | + sudo apt-get install uuid-dev + make - name: test run: | sudo apt-get install tcl8.5 From 2a14dcc7d1d3f4329389e432e05c6d063eea3626 Mon Sep 17 00:00:00 2001 From: John Sully Date: Wed, 15 Apr 2020 22:39:47 -0400 Subject: [PATCH 223/225] more dependencies for CI Former-commit-id: 2d6d3741624bf8f36b3c6a9e0557d02837f9cdaa --- .github/workflows/ci.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0d46f0b03..f675dd4bb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,7 +9,7 @@ jobs: - uses: actions/checkout@v1 - name: make run: | - sudo apt-get install uuid-dev + sudo apt-get install uuid-dev libcurl4-openssl-dev make - name: test run: | @@ -20,7 +20,9 @@ jobs: runs-on: ubuntu-16.04 steps: - uses: actions/checkout@v1 - - name: make + - name: | + sudo apt-get install uuid-dev libcurl4-openssl-dev + make run: make build-macos-latest: From e09a2ac94a2c984aff68ffb1db7f049f1ae44546 Mon Sep 17 00:00:00 2001 From: John Sully Date: Wed, 15 Apr 2020 22:43:40 -0400 Subject: [PATCH 224/225] OS X build break Former-commit-id: 5e6b4ab99e6f2ad29577c8d4cc151284f0c652b4 --- .github/workflows/ci.yml | 4 ++-- src/zmalloc.cpp | 6 +----- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f675dd4bb..a19e267d3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,10 +20,10 @@ jobs: runs-on: ubuntu-16.04 steps: - uses: actions/checkout@v1 - - name: | + - name: make + run: sudo apt-get install uuid-dev libcurl4-openssl-dev make - run: make build-macos-latest: runs-on: macos-latest diff --git a/src/zmalloc.cpp b/src/zmalloc.cpp index e4d5792ea..e3d364a97 100644 --- a/src/zmalloc.cpp +++ b/src/zmalloc.cpp @@ -449,9 +449,6 @@ size_t zmalloc_get_smap_bytes_by_field(const char *field, long pid) { return bytes; } #else -<<<<<<< HEAD:src/zmalloc.cpp -size_t zmalloc_get_smap_bytes_by_field(const char *field, long pid) { -======= /* Get sum of the specified field from libproc api call. * As there are per page value basis we need to convert * them accordingly. @@ -459,7 +456,7 @@ size_t zmalloc_get_smap_bytes_by_field(const char *field, long pid) { * Note that AnonHugePages is a no-op as THP feature * is not supported in this platform */ -size_t zmalloc_get_smap_bytes_by_field(char *field, long pid) { +size_t zmalloc_get_smap_bytes_by_field(const char *field, long pid) { #if defined(__APPLE__) struct proc_regioninfo pri; if (proc_pidinfo(pid, PROC_PIDREGIONINFO, 0, &pri, PROC_PIDREGIONINFO_SIZE) == @@ -474,7 +471,6 @@ size_t zmalloc_get_smap_bytes_by_field(char *field, long pid) { } return 0; #endif ->>>>>>> redis/6.0:src/zmalloc.c ((void) field); ((void) pid); return 0; From cf9efc8e9fa4c353bd635e36dcc92ee200e7a2ee Mon Sep 17 00:00:00 2001 From: John Sully Date: Wed, 15 Apr 2020 23:04:04 -0400 Subject: [PATCH 225/225] Deprecate travis and finish setting up github CI Former-commit-id: 90094a2dcaa004437c43818b411d44e722da420a --- .github/workflows/ci.yml | 2 +- .travis.yml | 29 ----------------------------- src/fastlock.cpp | 1 + 3 files changed, 2 insertions(+), 30 deletions(-) delete mode 100644 .travis.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a19e267d3..d812760c9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,7 +21,7 @@ jobs: steps: - uses: actions/checkout@v1 - name: make - run: + run: | sudo apt-get install uuid-dev libcurl4-openssl-dev make diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 8d20ca462..000000000 --- a/.travis.yml +++ /dev/null @@ -1,29 +0,0 @@ -language: generic -os: osx -matrix: - include: - - os: linux - script: make - env: COMPILER_NAME=g++-6 CXX=g++-6 CC=gcc-6 - addons: - apt: - packages: - - g++-6 - - nasm - - uuid-dev - sources: &sources - - llvm-toolchain-precise-3.8 - - ubuntu-toolchain-r-test - - os: linux - script: make MALLOC=libc - env: COMPILER_NAME=clang CXX=clang++-3.8 CC=clang-3.8 CXXFLAGS="-I/usr/include/libcxxabi/" LDFLAGS="-lc++" - addons: - apt: - packages: - - libc++abi-dev - - clang-3.8 - - libc++-dev - - libc++abi-dev - - nasm - - uuid-dev - sources: *sources diff --git a/src/fastlock.cpp b/src/fastlock.cpp index 5fe5c59db..a1cacfecd 100644 --- a/src/fastlock.cpp +++ b/src/fastlock.cpp @@ -27,6 +27,7 @@ * POSSIBILITY OF SUCH DAMAGE. */ +#include "fmacros.h" #include "fastlock.h" #include #include