From 010c2f136cc63e24a0385d2436be08d83b9f8cc8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BC=AF=E6=88=90?= Date: Thu, 1 Mar 2018 11:46:56 +0800 Subject: [PATCH 0001/1280] Boost up performance for redis PUB-SUB patterns matching If lots of clients PSUBSCRIBE to same patterns, multiple pattens matching will take place. This commit change it into just one single pattern matching by using a `dict *` to store the unique pattern and which clients subscribe to it. --- src/pubsub.c | 58 +++++++++++++++++++++++++++++++++++++++++----------- src/server.c | 1 + src/server.h | 1 + 3 files changed, 48 insertions(+), 12 deletions(-) diff --git a/src/pubsub.c b/src/pubsub.c index d1fffa20a..bbcfc1f43 100644 --- a/src/pubsub.c +++ b/src/pubsub.c @@ -125,6 +125,8 @@ int pubsubUnsubscribeChannel(client *c, robj *channel, int notify) { /* Subscribe a client to a pattern. Returns 1 if the operation succeeded, or 0 if the client was already subscribed to that pattern. */ int pubsubSubscribePattern(client *c, robj *pattern) { + dictEntry *de; + list *clients; int retval = 0; if (listSearchKey(c->pubsub_patterns,pattern) == NULL) { @@ -136,6 +138,16 @@ int pubsubSubscribePattern(client *c, robj *pattern) { pat->pattern = getDecodedObject(pattern); pat->client = c; listAddNodeTail(server.pubsub_patterns,pat); + /* Add the client to the pattern -> list of clients hash table */ + de = dictFind(server.pubsub_patterns_dict,pattern); + if (de == NULL) { + clients = listCreate(); + dictAdd(server.pubsub_patterns_dict,pattern,clients); + incrRefCount(pattern); + } else { + clients = dictGetVal(de); + } + listAddNodeTail(clients,c); } /* Notify the client */ addReply(c,shared.mbulkhdr[3]); @@ -148,6 +160,8 @@ int pubsubSubscribePattern(client *c, robj *pattern) { /* Unsubscribe a client from a channel. Returns 1 if the operation succeeded, or * 0 if the client was not subscribed to the specified channel. */ int pubsubUnsubscribePattern(client *c, robj *pattern, int notify) { + dictEntry *de; + list *clients; listNode *ln; pubsubPattern pat; int retval = 0; @@ -160,6 +174,18 @@ int pubsubUnsubscribePattern(client *c, robj *pattern, int notify) { pat.pattern = pattern; ln = listSearchKey(server.pubsub_patterns,&pat); listDelNode(server.pubsub_patterns,ln); + /* Remove the client from the pattern -> clients list hash table */ + de = dictFind(server.pubsub_patterns_dict,pattern); + serverAssertWithInfo(c,NULL,de != NULL); + clients = dictGetVal(de); + ln = listSearchKey(clients,c); + serverAssertWithInfo(c,NULL,ln != NULL); + listDelNode(clients,ln); + if (listLength(clients) == 0) { + /* Free the list and associated hash entry at all if this was + * the latest client. */ + dictDelete(server.pubsub_patterns_dict,pattern); + } } /* Notify the client */ if (notify) { @@ -225,6 +251,7 @@ int pubsubUnsubscribeAllPatterns(client *c, int notify) { int pubsubPublishMessage(robj *channel, robj *message) { int receivers = 0; dictEntry *de; + dictIterator *di; listNode *ln; listIter li; @@ -247,25 +274,32 @@ int pubsubPublishMessage(robj *channel, robj *message) { } } /* Send to clients listening to matching channels */ - if (listLength(server.pubsub_patterns)) { - listRewind(server.pubsub_patterns,&li); + di = dictGetIterator(server.pubsub_patterns_dict); + if (di) { channel = getDecodedObject(channel); - while ((ln = listNext(&li)) != NULL) { - pubsubPattern *pat = ln->value; - - if (stringmatchlen((char*)pat->pattern->ptr, - sdslen(pat->pattern->ptr), + while((de = dictNext(di)) != NULL) { + robj *pattern = dictGetKey(de); + list *clients = dictGetVal(de); + if (!stringmatchlen((char*)pattern->ptr, + sdslen(pattern->ptr), (char*)channel->ptr, sdslen(channel->ptr),0)) { - addReply(pat->client,shared.mbulkhdr[4]); - addReply(pat->client,shared.pmessagebulk); - addReplyBulk(pat->client,pat->pattern); - addReplyBulk(pat->client,channel); - addReplyBulk(pat->client,message); + continue; + } + listRewind(clients,&li); + while ((ln = listNext(&li)) != NULL) { + client *c = listNodeValue(ln); + + addReply(c,shared.mbulkhdr[4]); + addReply(c,shared.pmessagebulk); + addReplyBulk(c,pattern); + addReplyBulk(c,channel); + addReplyBulk(c,message); receivers++; } } decrRefCount(channel); + dictReleaseIterator(di); } return receivers; } diff --git a/src/server.c b/src/server.c index 1a6f30381..a071fceed 100644 --- a/src/server.c +++ b/src/server.c @@ -1900,6 +1900,7 @@ void initServer(void) { evictionPoolAlloc(); /* Initialize the LRU keys pool. */ server.pubsub_channels = dictCreate(&keylistDictType,NULL); server.pubsub_patterns = listCreate(); + server.pubsub_patterns_dict = dictCreate(&keylistDictType,NULL); listSetFreeMethod(server.pubsub_patterns,freePubsubPattern); listSetMatchMethod(server.pubsub_patterns,listMatchPubsubPattern); server.cronloops = 0; diff --git a/src/server.h b/src/server.h index 29919f5ee..a56663495 100644 --- a/src/server.h +++ b/src/server.h @@ -1163,6 +1163,7 @@ struct redisServer { /* Pubsub */ dict *pubsub_channels; /* Map channels to list of subscribed clients */ list *pubsub_patterns; /* A list of pubsub_patterns */ + dict *pubsub_patterns_dict; /* A dict of pubsub_patterns */ int notify_keyspace_events; /* Events to propagate via Pub/Sub. This is an xor of NOTIFY_... flags. */ /* Cluster */ From 27dc71f8dce4dfda5de8278c67075f9ad95fef28 Mon Sep 17 00:00:00 2001 From: "zhaozhao.zz" Date: Tue, 21 May 2019 11:37:13 +0800 Subject: [PATCH 0002/1280] Threaded IO: use main thread to handle write work --- src/networking.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/networking.c b/src/networking.c index 4bc22120a..4da762a15 100644 --- a/src/networking.c +++ b/src/networking.c @@ -2525,7 +2525,7 @@ pthread_mutex_t io_threads_mutex[IO_THREADS_MAX_NUM]; _Atomic unsigned long io_threads_pending[IO_THREADS_MAX_NUM]; int io_threads_active; /* Are the threads currently spinning waiting I/O? */ int io_threads_op; /* IO_THREADS_OP_WRITE or IO_THREADS_OP_READ. */ -list *io_threads_list[IO_THREADS_MAX_NUM]; +list *io_threads_list[IO_THREADS_MAX_NUM+1]; void *IOThreadMain(void *myid) { /* The ID is the thread number (from 0 to server.iothreads_num-1), and is @@ -2598,6 +2598,7 @@ void initThreadedIO(void) { } io_threads[i] = tid; } + io_threads_list[server.io_threads_num] = listCreate(); } void startThreadedIO(void) { @@ -2669,7 +2670,7 @@ int handleClientsWithPendingWritesUsingThreads(void) { while((ln = listNext(&li))) { client *c = listNodeValue(ln); c->flags &= ~CLIENT_PENDING_WRITE; - int target_id = item_id % server.io_threads_num; + int target_id = item_id % (server.io_threads_num+1); listAddNodeTail(io_threads_list[target_id],c); item_id++; } @@ -2682,6 +2683,13 @@ int handleClientsWithPendingWritesUsingThreads(void) { io_threads_pending[j] = count; } + listRewind(io_threads_list[server.io_threads_num],&li); + while((ln = listNext(&li))) { + client *c = listNodeValue(ln); + writeToClient(c->fd,c,0); + } + listEmpty(io_threads_list[server.io_threads_num]); + /* Wait for all threads to end their work. */ while(1) { unsigned long pending = 0; From 4517ac981f3352a95564d891907871e6464a5c5a Mon Sep 17 00:00:00 2001 From: "zhaozhao.zz" Date: Tue, 21 May 2019 11:42:10 +0800 Subject: [PATCH 0003/1280] Threaded IO: use main thread to handle read work --- src/networking.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/networking.c b/src/networking.c index 4da762a15..c2ae35181 100644 --- a/src/networking.c +++ b/src/networking.c @@ -2755,7 +2755,7 @@ int handleClientsWithPendingReadsUsingThreads(void) { int item_id = 0; while((ln = listNext(&li))) { client *c = listNodeValue(ln); - int target_id = item_id % server.io_threads_num; + int target_id = item_id % (server.io_threads_num+1); listAddNodeTail(io_threads_list[target_id],c); item_id++; } @@ -2768,6 +2768,13 @@ int handleClientsWithPendingReadsUsingThreads(void) { io_threads_pending[j] = count; } + listRewind(io_threads_list[server.io_threads_num],&li); + while((ln = listNext(&li))) { + client *c = listNodeValue(ln); + readQueryFromClient(NULL,c->fd,c,0); + } + listEmpty(io_threads_list[server.io_threads_num]); + /* Wait for all threads to end their work. */ while(1) { unsigned long pending = 0; From aeba242cb5ea04a76249d988358747d85637b6f1 Mon Sep 17 00:00:00 2001 From: Jamie Scott <5336227+IAmATeaPot418@users.noreply.github.com> Date: Wed, 16 Oct 2019 13:31:19 -0700 Subject: [PATCH 0004/1280] Update to directive in redis.conf (missing s) The directive tls-prefer-server-cipher is actually tls-prefer-server-ciphers in config.c. This results in a failed directive call shown below. This pull request adds the "s" in ciphers so that the directive is able to be properly called in config.c ubuntu@ip-172-31-16-31:~/redis$ src/redis-server ./redis.conf *** FATAL CONFIG FILE ERROR *** Reading the configuration file, at line 200 >>> 'tls-prefer-server-cipher yes' Bad directive or wrong number of arguments --- redis.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/redis.conf b/redis.conf index 408426f15..6e02cfed7 100644 --- a/redis.conf +++ b/redis.conf @@ -197,7 +197,7 @@ tcp-keepalive 300 # When choosing a cipher, use the server's preference instead of the client # preference. By default, the server follows the client's preference. # -# tls-prefer-server-cipher yes +# tls-prefer-server-ciphers yes ################################# GENERAL ##################################### From c574f5b6a2c4770e2abe185b546520b6e0586811 Mon Sep 17 00:00:00 2001 From: Guy Benoish Date: Tue, 5 Nov 2019 19:23:37 +0530 Subject: [PATCH 0005/1280] Make sure Redis does not reply with negative zero --- src/util.c | 4 ++++ tests/unit/type/incr.tcl | 6 ++++++ 2 files changed, 10 insertions(+) diff --git a/src/util.c b/src/util.c index 062a572d4..80b5a2453 100644 --- a/src/util.c +++ b/src/util.c @@ -560,6 +560,10 @@ int ld2string(char *buf, size_t len, long double value, ld2string_mode mode) { } if (*p == '.') l--; } + if (l == 2 && buf[0] == '-' && buf[1] == '0') { + buf[0] = '0'; + l = 1; + } break; default: return 0; /* Invalid mode. */ } diff --git a/tests/unit/type/incr.tcl b/tests/unit/type/incr.tcl index a58710d39..63bf2e116 100644 --- a/tests/unit/type/incr.tcl +++ b/tests/unit/type/incr.tcl @@ -151,4 +151,10 @@ start_server {tags {"incr"}} { catch {r incrbyfloat foo 1} err format $err } {ERR*valid*} + + test {No negative zero} { + r incrbyfloat foo [expr double(1)/41] + r incrbyfloat foo [expr double(-1)/41] + r get foo + } {0} } From d6ab82cb1780240064c3a51ae0f1061f3578fdf8 Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Tue, 10 Dec 2019 11:16:13 +0200 Subject: [PATCH 0006/1280] fix possible warning on incomplete struct init --- src/module.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/module.c b/src/module.c index 111e1c202..096a0bf2c 100644 --- a/src/module.c +++ b/src/module.c @@ -5652,7 +5652,7 @@ sds modulesCollectInfo(sds info, const char *section, int for_crash_report, int struct RedisModule *module = dictGetVal(de); if (!module->info_cb) continue; - RedisModuleInfoCtx info_ctx = {module, section, info, sections, 0}; + RedisModuleInfoCtx info_ctx = {module, section, info, sections, 0, 0}; module->info_cb(&info_ctx, for_crash_report); /* Implicitly end dicts (no way to handle errors, and we must add the newline). */ if (info_ctx.in_dict_field) From 1223fca94160c300f59cf0c1506925a17215f025 Mon Sep 17 00:00:00 2001 From: Guy Benoish Date: Wed, 18 Dec 2019 12:27:03 +0530 Subject: [PATCH 0007/1280] streamReplyWithRangeFromConsumerPEL: Redundant streamDecodeID --- src/t_stream.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/t_stream.c b/src/t_stream.c index a499f7381..c35c9d1be 100644 --- a/src/t_stream.c +++ b/src/t_stream.c @@ -1043,9 +1043,7 @@ size_t streamReplyWithRangeFromConsumerPEL(client *c, stream *s, streamID *start * by the user by other means. In that case we signal it emitting * the ID but then a NULL entry for the fields. */ addReplyArrayLen(c,2); - streamID id; - streamDecodeID(ri.key,&id); - addReplyStreamID(c,&id); + addReplyStreamID(c,&thisid); addReplyNullArray(c); } else { streamNACK *nack = ri.data; From 498f4838906a0661e3bd3ffd1ad7b95b57598755 Mon Sep 17 00:00:00 2001 From: "zhaozhao.zz" Date: Wed, 17 Jul 2019 11:00:51 +0800 Subject: [PATCH 0008/1280] lazyfree: add a new configuration lazyfree-lazy-user-del Delete keys in async way when executing DEL command, if lazyfree-lazy-user-del is yes. --- redis.conf | 5 ++++- src/config.c | 1 + src/db.c | 2 +- src/server.h | 1 + 4 files changed, 7 insertions(+), 2 deletions(-) diff --git a/redis.conf b/redis.conf index 870849a79..a0e1b5aa9 100644 --- a/redis.conf +++ b/redis.conf @@ -924,7 +924,9 @@ replica-priority 100 # or SORT with STORE option may delete existing keys. The SET command # itself removes any old content of the specified key in order to replace # it with the specified string. -# 4) During replication, when a replica performs a full resynchronization with +# 4) The DEL command itself, and normally it's not easy to replace DEL with +# UNLINK in user's codes. +# 5) During replication, when a replica performs a full resynchronization with # its master, the content of the whole database is removed in order to # load the RDB file just transferred. # @@ -936,6 +938,7 @@ replica-priority 100 lazyfree-lazy-eviction no lazyfree-lazy-expire no lazyfree-lazy-server-del no +lazyfree-lazy-user-del no replica-lazy-flush no ############################## APPEND ONLY MODE ############################### diff --git a/src/config.c b/src/config.c index 3d53e656b..0bb2fc237 100644 --- a/src/config.c +++ b/src/config.c @@ -2147,6 +2147,7 @@ standardConfig configs[] = { createBoolConfig("lazyfree-lazy-eviction", NULL, MODIFIABLE_CONFIG, server.lazyfree_lazy_eviction, 0, NULL, NULL), createBoolConfig("lazyfree-lazy-expire", NULL, MODIFIABLE_CONFIG, server.lazyfree_lazy_expire, 0, NULL, NULL), createBoolConfig("lazyfree-lazy-server-del", NULL, MODIFIABLE_CONFIG, server.lazyfree_lazy_server_del, 0, NULL, NULL), + createBoolConfig("lazyfree-lazy-user-del", NULL, MODIFIABLE_CONFIG, server.lazyfree_lazy_user_del , 0, NULL, NULL), createBoolConfig("repl-disable-tcp-nodelay", NULL, MODIFIABLE_CONFIG, server.repl_disable_tcp_nodelay, 0, NULL, NULL), createBoolConfig("repl-diskless-sync", NULL, MODIFIABLE_CONFIG, server.repl_diskless_sync, 0, NULL, NULL), createBoolConfig("gopher-enabled", NULL, MODIFIABLE_CONFIG, server.gopher_enabled, 0, NULL, NULL), diff --git a/src/db.c b/src/db.c index 483095a7c..88102e557 100644 --- a/src/db.c +++ b/src/db.c @@ -536,7 +536,7 @@ void delGenericCommand(client *c, int lazy) { } void delCommand(client *c) { - delGenericCommand(c,0); + delGenericCommand(c,server.lazyfree_lazy_user_del); } void unlinkCommand(client *c) { diff --git a/src/server.h b/src/server.h index 7a78c884f..8c25534fe 100644 --- a/src/server.h +++ b/src/server.h @@ -1375,6 +1375,7 @@ struct redisServer { int lazyfree_lazy_eviction; int lazyfree_lazy_expire; int lazyfree_lazy_server_del; + int lazyfree_lazy_user_del; /* Latency monitor */ long long latency_monitor_threshold; dict *latency_events; From eaf450c3011c2a28ac2b885bed677539115e837b Mon Sep 17 00:00:00 2001 From: Johannes Truschnigg Date: Thu, 19 Dec 2019 21:47:24 +0100 Subject: [PATCH 0009/1280] Signal systemd readiness atfer Partial Resync "Partial Resynchronization" is a special variant of replication success that we have to tell systemd about if it is managing redis-server via a Type=Notify service unit. --- src/replication.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/replication.c b/src/replication.c index 68dc77a61..01619810c 100644 --- a/src/replication.c +++ b/src/replication.c @@ -2172,6 +2172,10 @@ void syncWithMaster(connection *conn) { if (psync_result == PSYNC_CONTINUE) { serverLog(LL_NOTICE, "MASTER <-> REPLICA sync: Master accepted a Partial Resynchronization."); + if (server.supervised_mode == SUPERVISED_SYSTEMD) { + redisCommunicateSystemd("STATUS=MASTER <-> REPLICA sync: Partial Resynchronization accepted. Ready to accept connections.\n"); + redisCommunicateSystemd("READY=1\n"); + } return; } From 1d353f171b4e541f5802cf5b9982d2bcdd2df77d Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 20 Dec 2019 12:29:02 +0100 Subject: [PATCH 0010/1280] Fix ip and missing mode in RM_GetClusterNodeInfo(). --- src/module.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/module.c b/src/module.c index 33b69f8c2..705ffec2c 100644 --- a/src/module.c +++ b/src/module.c @@ -5082,10 +5082,13 @@ int RM_GetClusterNodeInfo(RedisModuleCtx *ctx, const char *id, char *ip, char *m UNUSED(ctx); clusterNode *node = clusterLookupNode(id); - if (node->flags & (CLUSTER_NODE_NOADDR|CLUSTER_NODE_HANDSHAKE)) + if (node == NULL || + node->flags & (CLUSTER_NODE_NOADDR|CLUSTER_NODE_HANDSHAKE)) + { return REDISMODULE_ERR; + } - if (ip) memcpy(ip,node->name,REDISMODULE_NODE_ID_LEN); + if (ip) strncpy(ip,node->ip,NET_IP_STR_LEN); if (master_id) { /* If the information is not available, the function will set the From 9cbabf32a5f8be1208f924db4896564fa026e824 Mon Sep 17 00:00:00 2001 From: "bodong.ybd" Date: Sat, 21 Dec 2019 21:27:38 +0800 Subject: [PATCH 0011/1280] Fix spop return nil #4709 --- src/t_set.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/t_set.c b/src/t_set.c index abbf82fde..60cf22d8c 100644 --- a/src/t_set.c +++ b/src/t_set.c @@ -415,7 +415,7 @@ void spopWithCountCommand(client *c) { /* Make sure a key with the name inputted exists, and that it's type is * indeed a set. Otherwise, return nil */ - if ((set = lookupKeyWriteOrReply(c,c->argv[1],shared.null[c->resp])) + if ((set = lookupKeyWriteOrReply(c,c->argv[1],shared.emptyset[c->resp])) == NULL || checkType(c,set,OBJ_SET)) return; /* If count is zero, serve an empty set ASAP to avoid special From e72ad3483a438f8751928dbd3aa14bb464a73111 Mon Sep 17 00:00:00 2001 From: Khem Raj Date: Sat, 21 Dec 2019 11:17:50 -0800 Subject: [PATCH 0012/1280] Mark extern definition of SDS_NOINIT in sds.h This helps in avoiding multiple definition of this variable, its also defined globally in sds.c Signed-off-by: Khem Raj --- src/sds.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sds.h b/src/sds.h index 1bdb60dec..adcc12c0a 100644 --- a/src/sds.h +++ b/src/sds.h @@ -34,7 +34,7 @@ #define __SDS_H #define SDS_MAX_PREALLOC (1024*1024) -const char *SDS_NOINIT; +extern const char *SDS_NOINIT; #include #include From 5cc62cc219f7b5a6a1dbd0a0e8cb4d8b905e48e1 Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Mon, 23 Dec 2019 10:15:52 +0200 Subject: [PATCH 0013/1280] modules don't signalModifiedKey in setKey() since that's done (optionally) in RM_CloseKey --- src/db.c | 6 +++--- src/module.c | 6 +++--- src/server.h | 2 +- src/t_string.c | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/db.c b/src/db.c index 32bb35652..bb93a6071 100644 --- a/src/db.c +++ b/src/db.c @@ -220,7 +220,7 @@ void dbOverwrite(redisDb *db, robj *key, robj *val) { * unless 'keepttl' is true. * * All the new keys in the database should be created via this interface. */ -void genericSetKey(redisDb *db, robj *key, robj *val, int keepttl) { +void genericSetKey(redisDb *db, robj *key, robj *val, int keepttl, int signal) { if (lookupKeyWrite(db,key) == NULL) { dbAdd(db,key,val); } else { @@ -228,12 +228,12 @@ void genericSetKey(redisDb *db, robj *key, robj *val, int keepttl) { } incrRefCount(val); if (!keepttl) removeExpire(db,key); - signalModifiedKey(db,key); + if (signal) signalModifiedKey(db,key); } /* Common case for genericSetKey() where the TTL is not retained. */ void setKey(redisDb *db, robj *key, robj *val) { - genericSetKey(db,key,val,0); + genericSetKey(db,key,val,0,1); } /* Return true if the specified key exists in the specified database. diff --git a/src/module.c b/src/module.c index 705ffec2c..afeb9c046 100644 --- a/src/module.c +++ b/src/module.c @@ -2128,7 +2128,7 @@ RedisModuleString *RM_RandomKey(RedisModuleCtx *ctx) { int RM_StringSet(RedisModuleKey *key, RedisModuleString *str) { if (!(key->mode & REDISMODULE_WRITE) || key->iter) return REDISMODULE_ERR; RM_DeleteKey(key); - setKey(key->db,key->key,str); + genericSetKey(key->db,key->key,str,0,0); key->value = str; return REDISMODULE_OK; } @@ -2208,7 +2208,7 @@ int RM_StringTruncate(RedisModuleKey *key, size_t newlen) { if (key->value == NULL) { /* Empty key: create it with the new size. */ robj *o = createObject(OBJ_STRING,sdsnewlen(NULL, newlen)); - setKey(key->db,key->key,o); + genericSetKey(key->db,key->key,o,0,0); key->value = o; decrRefCount(o); } else { @@ -3597,7 +3597,7 @@ int RM_ModuleTypeSetValue(RedisModuleKey *key, moduleType *mt, void *value) { if (!(key->mode & REDISMODULE_WRITE) || key->iter) return REDISMODULE_ERR; RM_DeleteKey(key); robj *o = createModuleObject(mt,value); - setKey(key->db,key->key,o); + genericSetKey(key->db,key->key,o,0,0); decrRefCount(o); key->value = o; return REDISMODULE_OK; diff --git a/src/server.h b/src/server.h index 8e354c03d..c6d40f704 100644 --- a/src/server.h +++ b/src/server.h @@ -2032,7 +2032,7 @@ int objectSetLRUOrLFU(robj *val, long long lfu_freq, long long lru_idle, #define LOOKUP_NOTOUCH (1<<0) void dbAdd(redisDb *db, robj *key, robj *val); void dbOverwrite(redisDb *db, robj *key, robj *val); -void genericSetKey(redisDb *db, robj *key, robj *val, int keepttl); +void genericSetKey(redisDb *db, robj *key, robj *val, int keepttl, int signal); void setKey(redisDb *db, robj *key, robj *val); int dbExists(redisDb *db, robj *key); robj *dbRandomKey(redisDb *db); diff --git a/src/t_string.c b/src/t_string.c index 8ccd69eb9..3174e9ccd 100644 --- a/src/t_string.c +++ b/src/t_string.c @@ -84,7 +84,7 @@ void setGenericCommand(client *c, int flags, robj *key, robj *val, robj *expire, addReply(c, abort_reply ? abort_reply : shared.null[c->resp]); return; } - genericSetKey(c->db,key,val,flags & OBJ_SET_KEEPTTL); + genericSetKey(c->db,key,val,flags & OBJ_SET_KEEPTTL,1); server.dirty++; if (expire) setExpire(c,c->db,key,mstime()+milliseconds); notifyKeyspaceEvent(NOTIFY_STRING,"set",key,c->db->id); From 7ca3d5db5f7fac74b84c39998b653893aced373e Mon Sep 17 00:00:00 2001 From: Yossi Gottlieb Date: Thu, 6 Jun 2019 20:08:26 +0300 Subject: [PATCH 0014/1280] Add REDISMODULE_CTX_FLAGS_MULTI_DIRTY. --- src/module.c | 6 ++++++ src/redismodule.h | 2 ++ 2 files changed, 8 insertions(+) diff --git a/src/module.c b/src/module.c index 705ffec2c..965bb4460 100644 --- a/src/module.c +++ b/src/module.c @@ -1842,6 +1842,12 @@ int RM_GetContextFlags(RedisModuleCtx *ctx) { flags |= REDISMODULE_CTX_FLAGS_REPLICATED; } + /* For DIRTY flags, we need the blocked client if used */ + client *c = ctx->blocked_client ? ctx->blocked_client->client : ctx->client; + if (c && (c->flags & (CLIENT_DIRTY_CAS|CLIENT_DIRTY_EXEC))) { + flags |= REDISMODULE_CTX_FLAGS_MULTI_DIRTY; + } + if (server.cluster_enabled) flags |= REDISMODULE_CTX_FLAGS_CLUSTER; diff --git a/src/redismodule.h b/src/redismodule.h index 57f98fb93..637078f2b 100644 --- a/src/redismodule.h +++ b/src/redismodule.h @@ -110,6 +110,8 @@ #define REDISMODULE_CTX_FLAGS_REPLICA_IS_ONLINE (1<<17) /* There is currently some background process active. */ #define REDISMODULE_CTX_FLAGS_ACTIVE_CHILD (1<<18) +/* The next EXEC will fail due to dirty CAS (touched keys). */ +#define REDISMODULE_CTX_FLAGS_MULTI_DIRTY (1<<19) /* Keyspace changes notification classes. Every class is associated with a * character for configuration purposes. From 6731e0dab5b99cfdedef3552a5cabc4a718a737c Mon Sep 17 00:00:00 2001 From: Guy Benoish Date: Thu, 26 Dec 2019 15:31:37 +0530 Subject: [PATCH 0015/1280] Stream: Handle streamID-related edge cases This commit solves several edge cases that are related to exhausting the streamID limits: We should correctly calculate the succeeding streamID instead of blindly incrementing 'seq' This affects both XREAD and XADD. Other (unrelated) changes: Reply with a better error message when trying to add an entry to a stream that has exhausted last_id --- src/blocked.c | 2 +- src/stream.h | 1 + src/t_stream.c | 28 +++++++++++++++++++++++++--- tests/unit/type/stream.tcl | 27 +++++++++++++++++++++++++++ 4 files changed, 54 insertions(+), 4 deletions(-) diff --git a/src/blocked.c b/src/blocked.c index 20c0e760a..06aa5850e 100644 --- a/src/blocked.c +++ b/src/blocked.c @@ -388,7 +388,7 @@ void serveClientsBlockedOnStreamKey(robj *o, readyList *rl) { if (streamCompareID(&s->last_id, gt) > 0) { streamID start = *gt; - start.seq++; /* Can't overflow, it's an uint64_t */ + streamIncrID(&start); /* Lookup the consumer for the group, if any. */ streamConsumer *consumer = NULL; diff --git a/src/stream.h b/src/stream.h index 7de769ba1..b69073994 100644 --- a/src/stream.h +++ b/src/stream.h @@ -111,5 +111,6 @@ streamNACK *streamCreateNACK(streamConsumer *consumer); void streamDecodeID(void *buf, streamID *id); int streamCompareID(streamID *a, streamID *b); void streamFreeNACK(streamNACK *na); +void streamIncrID(streamID *id); #endif diff --git a/src/t_stream.c b/src/t_stream.c index a499f7381..3d46ca0da 100644 --- a/src/t_stream.c +++ b/src/t_stream.c @@ -73,6 +73,21 @@ unsigned long streamLength(const robj *subject) { return s->length; } +/* Set 'id' to be its successor streamID */ +void streamIncrID(streamID *id) { + if (id->seq == UINT64_MAX) { + if (id->ms == UINT64_MAX) { + /* Special case where 'id' is the last possible streamID... */ + id->ms = id->seq = 0; + } else { + id->ms++; + id->seq = 0; + } + } else { + id->seq++; + } +} + /* Generate the next stream item ID given the previous one. If the current * milliseconds Unix time is greater than the previous one, just use this * as time part and start with sequence part of zero. Otherwise we use the @@ -83,8 +98,8 @@ void streamNextID(streamID *last_id, streamID *new_id) { new_id->ms = ms; new_id->seq = 0; } else { - new_id->ms = last_id->ms; - new_id->seq = last_id->seq+1; + *new_id = *last_id; + streamIncrID(new_id); } } @@ -1220,6 +1235,13 @@ void xaddCommand(client *c) { if ((o = streamTypeLookupWriteOrCreate(c,c->argv[1])) == NULL) return; s = o->ptr; + /* Return ASAP if the stream has reached the last possible ID */ + if (s->last_id.ms == UINT64_MAX && s->last_id.seq == UINT64_MAX) { + addReplyError(c,"The stream has exhausted the last possible ID, " + "unable to add more items"); + return; + } + /* Append using the low level function and return the ID. */ if (streamAppendItem(s,c->argv+field_pos,(c->argc-field_pos)/2, &id, id_given ? &id : NULL) @@ -1509,7 +1531,7 @@ void xreadCommand(client *c) { * so start from the next ID, since we want only messages with * IDs greater than start. */ streamID start = *gt; - start.seq++; /* uint64_t can't overflow in this context. */ + streamIncrID(&start); /* Emit the two elements sub-array consisting of the name * of the stream and the data we extracted from it. */ diff --git a/tests/unit/type/stream.tcl b/tests/unit/type/stream.tcl index 656bac5de..9840e3b74 100644 --- a/tests/unit/type/stream.tcl +++ b/tests/unit/type/stream.tcl @@ -328,6 +328,33 @@ start_server { assert_equal [r xrevrange teststream2 1234567891245 -] {{1234567891240-0 {key1 value2}} {1234567891230-0 {key1 value1}}} } + + test {XREAD streamID edge (no-blocking)} { + r del x + r XADD x 1-1 f v + r XADD x 1-18446744073709551615 f v + r XADD x 2-1 f v + set res [r XREAD BLOCK 0 STREAMS x 1-18446744073709551615] + assert {[lindex $res 0 1 0] == {2-1 {f v}}} + } + + test {XREAD streamID edge (blocking)} { + r del x + set rd [redis_deferring_client] + $rd XREAD BLOCK 0 STREAMS x 1-18446744073709551615 + r XADD x 1-1 f v + r XADD x 1-18446744073709551615 f v + r XADD x 2-1 f v + set res [$rd read] + assert {[lindex $res 0 1 0] == {2-1 {f v}}} + } + + test {XADD streamID edge} { + r del x + r XADD x 2577343934890-18446744073709551615 f v ;# we need the timestamp to be in the future + r XADD x * f2 v2 + assert_equal [r XRANGE x - +] {{2577343934890-18446744073709551615 {f v}} {2577343934891-0 {f2 v2}}} + } } start_server {tags {"stream"} overrides {appendonly yes}} { From f10051c5fdf48891b38999e8aab21520c9d9b9dc Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Thu, 26 Dec 2019 13:59:58 +0200 Subject: [PATCH 0016/1280] config.c adjust config limits and mutable - make lua-replicate-commands mutable (it never was, but i don't see why) - make tcp-backlog immutable (fix a recent refactory mistake) - increase the max limit of a few configs to match what they were before the recent refactory --- src/config.c | 12 ++++++------ tests/unit/introspection.tcl | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/config.c b/src/config.c index 3d53e656b..9aa847183 100644 --- a/src/config.c +++ b/src/config.c @@ -2137,7 +2137,7 @@ standardConfig configs[] = { createBoolConfig("rdbchecksum", NULL, IMMUTABLE_CONFIG, server.rdb_checksum, 1, NULL, NULL), createBoolConfig("daemonize", NULL, IMMUTABLE_CONFIG, server.daemonize, 0, NULL, NULL), createBoolConfig("io-threads-do-reads", NULL, IMMUTABLE_CONFIG, server.io_threads_do_reads, 0,NULL, NULL), /* Read + parse from threads? */ - createBoolConfig("lua-replicate-commands", NULL, IMMUTABLE_CONFIG, server.lua_always_replicate_commands, 1, NULL, NULL), + createBoolConfig("lua-replicate-commands", NULL, MODIFIABLE_CONFIG, server.lua_always_replicate_commands, 1, NULL, NULL), createBoolConfig("always-show-logo", NULL, IMMUTABLE_CONFIG, server.always_show_logo, 0, NULL, NULL), createBoolConfig("protected-mode", NULL, MODIFIABLE_CONFIG, server.protected_mode, 1, NULL, NULL), createBoolConfig("rdbcompression", NULL, MODIFIABLE_CONFIG, server.rdb_compression, 1, NULL, NULL), @@ -2209,7 +2209,7 @@ standardConfig configs[] = { createIntConfig("maxmemory-samples", NULL, MODIFIABLE_CONFIG, 1, INT_MAX, server.maxmemory_samples, 5, INTEGER_CONFIG, NULL, NULL), createIntConfig("timeout", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.maxidletime, 0, INTEGER_CONFIG, NULL, NULL), /* Default client timeout: infinite */ createIntConfig("replica-announce-port", "slave-announce-port", MODIFIABLE_CONFIG, 0, 65535, server.slave_announce_port, 0, INTEGER_CONFIG, NULL, NULL), - createIntConfig("tcp-backlog", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.tcp_backlog, 511, INTEGER_CONFIG, NULL, NULL), /* TCP listen backlog. */ + createIntConfig("tcp-backlog", NULL, IMMUTABLE_CONFIG, 0, INT_MAX, server.tcp_backlog, 511, INTEGER_CONFIG, NULL, NULL), /* TCP listen backlog. */ createIntConfig("cluster-announce-bus-port", NULL, MODIFIABLE_CONFIG, 0, 65535, server.cluster_announce_bus_port, 0, INTEGER_CONFIG, NULL, NULL), /* Default: Use +10000 offset. */ createIntConfig("cluster-announce-port", NULL, MODIFIABLE_CONFIG, 0, 65535, server.cluster_announce_port, 0, INTEGER_CONFIG, NULL, NULL), /* Use server.port */ createIntConfig("repl-timeout", NULL, MODIFIABLE_CONFIG, 1, INT_MAX, server.repl_timeout, 60, INTEGER_CONFIG, NULL, NULL), @@ -2235,9 +2235,9 @@ standardConfig configs[] = { createLongLongConfig("cluster-node-timeout", NULL, MODIFIABLE_CONFIG, 0, LLONG_MAX, server.cluster_node_timeout, 15000, INTEGER_CONFIG, NULL, NULL), createLongLongConfig("slowlog-log-slower-than", NULL, MODIFIABLE_CONFIG, -1, LLONG_MAX, server.slowlog_log_slower_than, 10000, INTEGER_CONFIG, NULL, NULL), createLongLongConfig("latency-monitor-threshold", NULL, MODIFIABLE_CONFIG, 0, LLONG_MAX, server.latency_monitor_threshold, 0, INTEGER_CONFIG, NULL, NULL), - createLongLongConfig("proto-max-bulk-len", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.proto_max_bulk_len, 512ll*1024*1024, MEMORY_CONFIG, NULL, NULL), /* Bulk request max size */ - createLongLongConfig("stream-node-max-entries", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.stream_node_max_entries, 100, INTEGER_CONFIG, NULL, NULL), - createLongLongConfig("repl-backlog-size", NULL, MODIFIABLE_CONFIG, 1, LONG_MAX, server.repl_backlog_size, 1024*1024, MEMORY_CONFIG, NULL, updateReplBacklogSize), /* Default: 1mb */ + createLongLongConfig("proto-max-bulk-len", NULL, MODIFIABLE_CONFIG, 0, LLONG_MAX, server.proto_max_bulk_len, 512ll*1024*1024, MEMORY_CONFIG, NULL, NULL), /* Bulk request max size */ + createLongLongConfig("stream-node-max-entries", NULL, MODIFIABLE_CONFIG, 0, LLONG_MAX, server.stream_node_max_entries, 100, INTEGER_CONFIG, NULL, NULL), + createLongLongConfig("repl-backlog-size", NULL, MODIFIABLE_CONFIG, 1, LLONG_MAX, server.repl_backlog_size, 1024*1024, MEMORY_CONFIG, NULL, updateReplBacklogSize), /* Default: 1mb */ /* Unsigned Long Long configs */ createULongLongConfig("maxmemory", NULL, MODIFIABLE_CONFIG, 0, ULLONG_MAX, server.maxmemory, 0, MEMORY_CONFIG, NULL, updateMaxmemory), @@ -2246,7 +2246,7 @@ standardConfig configs[] = { createSizeTConfig("hash-max-ziplist-entries", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.hash_max_ziplist_entries, 512, INTEGER_CONFIG, NULL, NULL), createSizeTConfig("set-max-intset-entries", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.set_max_intset_entries, 512, INTEGER_CONFIG, NULL, NULL), createSizeTConfig("zset-max-ziplist-entries", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.zset_max_ziplist_entries, 128, INTEGER_CONFIG, NULL, NULL), - createSizeTConfig("active-defrag-ignore-bytes", NULL, MODIFIABLE_CONFIG, 1, LONG_MAX, server.active_defrag_ignore_bytes, 100<<20, MEMORY_CONFIG, NULL, NULL), /* Default: don't defrag if frag overhead is below 100mb */ + createSizeTConfig("active-defrag-ignore-bytes", NULL, MODIFIABLE_CONFIG, 1, LLONG_MAX, server.active_defrag_ignore_bytes, 100<<20, MEMORY_CONFIG, NULL, NULL), /* Default: don't defrag if frag overhead is below 100mb */ createSizeTConfig("hash-max-ziplist-value", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.hash_max_ziplist_value, 64, MEMORY_CONFIG, NULL, NULL), createSizeTConfig("stream-node-max-bytes", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.stream_node_max_bytes, 4096, MEMORY_CONFIG, NULL, NULL), createSizeTConfig("zset-max-ziplist-value", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.zset_max_ziplist_value, 64, MEMORY_CONFIG, NULL, NULL), diff --git a/tests/unit/introspection.tcl b/tests/unit/introspection.tcl index 3bf3d367b..cd905084a 100644 --- a/tests/unit/introspection.tcl +++ b/tests/unit/introspection.tcl @@ -65,7 +65,7 @@ start_server {tags {"introspection"}} { rdbchecksum daemonize io-threads-do-reads - lua-replicate-commands + tcp-backlog always-show-logo syslog-enabled cluster-enabled From 27462130385331b8c454e64e68c7fe495188e042 Mon Sep 17 00:00:00 2001 From: antirez Date: Sun, 29 Dec 2019 15:40:40 +0100 Subject: [PATCH 0017/1280] Inline protocol: handle empty strings well. This bug is from the first version of Redis. Probably the problem here is that before we used an SDS split function that created empty strings for additional spaces, like in "SET foo bar". AFAIK later we replaced it with the curretn sdssplitarg() API that has no such a problem. As a result, we introduced a bug, where it is no longer possible to do something like: SET foo "" Using the inline protocol. Now it is fixed. --- src/networking.c | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/networking.c b/src/networking.c index 37f8fa9b7..dac45751d 100644 --- a/src/networking.c +++ b/src/networking.c @@ -1453,12 +1453,8 @@ int processInlineBuffer(client *c) { /* Create redis objects for all arguments. */ for (c->argc = 0, j = 0; j < argc; j++) { - if (sdslen(argv[j])) { - c->argv[c->argc] = createObject(OBJ_STRING,argv[j]); - c->argc++; - } else { - sdsfree(argv[j]); - } + c->argv[c->argc] = createObject(OBJ_STRING,argv[j]); + c->argc++; } zfree(argv); return C_OK; From c0d6c9a0c16ea9ea4d2335ebbbc2daad0fa8c33f Mon Sep 17 00:00:00 2001 From: antirez Date: Sun, 29 Dec 2019 15:44:59 +0100 Subject: [PATCH 0018/1280] Fix duplicated CLIENT SETNAME reply. Happened when we set the name to "" to cancel the name. Was introduced during the RESP3 refactoring. See #6036. --- src/networking.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/networking.c b/src/networking.c index dac45751d..a558ae91a 100644 --- a/src/networking.c +++ b/src/networking.c @@ -1998,7 +1998,6 @@ int clientSetNameOrReply(client *c, robj *name) { if (len == 0) { if (c->name) decrRefCount(c->name); c->name = NULL; - addReply(c,shared.ok); return C_OK; } From 13df9046094f07bd228a2f80b83bd1b32b8653d0 Mon Sep 17 00:00:00 2001 From: Guy Benoish Date: Thu, 26 Dec 2019 12:19:24 +0530 Subject: [PATCH 0019/1280] Blocking XREAD[GROUP] should always reply with valid data (or timeout) This commit solves the following bug: 127.0.0.1:6379> XGROUP CREATE x grp $ MKSTREAM OK 127.0.0.1:6379> XADD x 666 f v "666-0" 127.0.0.1:6379> XREADGROUP GROUP grp Alice BLOCK 0 STREAMS x > 1) 1) "x" 2) 1) 1) "666-0" 2) 1) "f" 2) "v" 127.0.0.1:6379> XADD x 667 f v "667-0" 127.0.0.1:6379> XDEL x 667 (integer) 1 127.0.0.1:6379> XREADGROUP GROUP grp Alice BLOCK 0 STREAMS x > 1) 1) "x" 2) (empty array) The root cause is that we use s->last_id in streamCompareID while we should use the last *valid* ID --- src/t_stream.c | 29 +++++++++++++++++++---------- tests/unit/type/stream-cgroups.tcl | 14 ++++++++++++++ tests/unit/type/stream.tcl | 11 +++++++++++ 3 files changed, 44 insertions(+), 10 deletions(-) diff --git a/src/t_stream.c b/src/t_stream.c index a499f7381..d1b39d2f3 100644 --- a/src/t_stream.c +++ b/src/t_stream.c @@ -782,6 +782,16 @@ int streamDeleteItem(stream *s, streamID *id) { return deleted; } +/* Get the last valid (non-tombstone) streamID of 's'. */ +void streamLastValidID(stream *s, streamID *maxid) +{ + streamIterator si; + streamIteratorStart(&si,s,NULL,NULL,1); + int64_t numfields; + streamIteratorGetID(&si,maxid,&numfields); + streamIteratorStop(&si); +} + /* Emit a reply in the client output buffer by formatting a Stream ID * in the standard - format, using the simple string protocol * of REPL. */ @@ -1484,20 +1494,23 @@ void xreadCommand(client *c) { { serve_synchronously = 1; serve_history = 1; - } else { + } else if (s->length) { /* We also want to serve a consumer in a consumer group * synchronously in case the group top item delivered is smaller * than what the stream has inside. */ - streamID *last = &groups[i]->last_id; - if (s->length && (streamCompareID(&s->last_id, last) > 0)) { + streamID maxid, *last = &groups[i]->last_id; + streamLastValidID(s, &maxid); + if (streamCompareID(&maxid, last) > 0) { serve_synchronously = 1; *gt = *last; } } - } else { + } else if (s->length) { /* For consumers without a group, we serve synchronously if we can * actually provide at least one item from the stream. */ - if (s->length && (streamCompareID(&s->last_id, gt) > 0)) { + streamID maxid; + streamLastValidID(s, &maxid); + if (streamCompareID(&maxid, gt) > 0) { serve_synchronously = 1; } } @@ -1849,11 +1862,7 @@ void xsetidCommand(client *c) { * item, otherwise the fundamental ID monotonicity assumption is violated. */ if (s->length > 0) { streamID maxid; - streamIterator si; - streamIteratorStart(&si,s,NULL,NULL,1); - int64_t numfields; - streamIteratorGetID(&si,&maxid,&numfields); - streamIteratorStop(&si); + streamLastValidID(s,&maxid); if (streamCompareID(&id,&maxid) < 0) { addReplyError(c,"The ID specified in XSETID is smaller than the " diff --git a/tests/unit/type/stream-cgroups.tcl b/tests/unit/type/stream-cgroups.tcl index 34d4061c2..a59e168ef 100644 --- a/tests/unit/type/stream-cgroups.tcl +++ b/tests/unit/type/stream-cgroups.tcl @@ -147,6 +147,20 @@ start_server { assert {[lindex $res 0 1 1] == {2-0 {field1 B}}} } + test {Blocking XREADGROUP will not reply with an empty array} { + r del mystream + r XGROUP CREATE mystream mygroup $ MKSTREAM + r XADD mystream 666 f v + set res [r XREADGROUP GROUP mygroup Alice BLOCK 10 STREAMS mystream ">"] + assert {[lindex $res 0 1 0] == {666-0 {f v}}} + r XADD mystream 667 f2 v2 + r XDEL mystream 667 + set rd [redis_deferring_client] + $rd XREADGROUP GROUP mygroup Alice BLOCK 10 STREAMS mystream ">" + after 20 + assert {[$rd read] == {}} ;# before the fix, client didn't even block, but was served synchronously with {mystream {}} + } + test {XCLAIM can claim PEL items from another consumer} { # Add 3 items into the stream, and create a consumer group r del mystream diff --git a/tests/unit/type/stream.tcl b/tests/unit/type/stream.tcl index 656bac5de..585e2a634 100644 --- a/tests/unit/type/stream.tcl +++ b/tests/unit/type/stream.tcl @@ -191,6 +191,17 @@ start_server { assert {[lindex $res 0 1 0 1] eq {old abcd1234}} } + test {Blocking XREAD will not reply with an empty array} { + r del s1 + r XADD s1 666 f v + r XADD s1 667 f2 v2 + r XDEL s1 667 + set rd [redis_deferring_client] + $rd XREAD BLOCK 10 STREAMS s1 666 + after 20 + assert {[$rd read] == {}} ;# before the fix, client didn't even block, but was served synchronously with {s1 {}} + } + test "XREAD: XADD + DEL should not awake client" { set rd [redis_deferring_client] r del s1 From fe651411355ee46ea5002150af33033b69db3e01 Mon Sep 17 00:00:00 2001 From: Guy Benoish Date: Tue, 24 Dec 2019 17:14:23 +0530 Subject: [PATCH 0020/1280] Modules: Fix blocked-client-related memory leak If a blocked module client times-out (or disconnects, unblocked by CLIENT command, etc.) we need to call moduleUnblockClient in order to free memory allocated by the module sub-system and blocked-client private data Other changes: Made blockedonkeys.tcl tests a bit more aggressive in order to smoke-out potential memory leaks --- src/module.c | 9 +++++++ tests/modules/blockonkeys.c | 13 ++++++----- tests/unit/moduleapi/blockonkeys.tcl | 35 ++++++++++++++++++++++++++++ 3 files changed, 51 insertions(+), 6 deletions(-) diff --git a/src/module.c b/src/module.c index 705ffec2c..cbab735f1 100644 --- a/src/module.c +++ b/src/module.c @@ -4271,6 +4271,15 @@ void unblockClientFromModule(client *c) { moduleFreeContext(&ctx); } + /* If we made it here and client is still blocked it means that the command + * timed-out, client was killed or disconnected and disconnect_callback was + * not implemented (or it was, but RM_UnblockClient was not called from + * within it, as it should). + * We must call moduleUnblockClient in order to free privdata and + * RedisModuleBlockedClient */ + if (!bc->unblocked) + moduleUnblockClient(c); + bc->client = NULL; /* Reset the client for a new query since, for blocking commands implemented * into modules, we do not it immediately after the command returns (and diff --git a/tests/modules/blockonkeys.c b/tests/modules/blockonkeys.c index 959918b1c..10dc65b1a 100644 --- a/tests/modules/blockonkeys.c +++ b/tests/modules/blockonkeys.c @@ -172,13 +172,13 @@ int bpopgt_reply_callback(RedisModuleCtx *ctx, RedisModuleString **argv, int arg REDISMODULE_NOT_USED(argv); REDISMODULE_NOT_USED(argc); RedisModuleString *keyname = RedisModule_GetBlockedClientReadyKey(ctx); - long long gt = (long long)RedisModule_GetBlockedClientPrivateData(ctx); + long long *pgt = RedisModule_GetBlockedClientPrivateData(ctx); fsl_t *fsl; if (!get_fsl(ctx, keyname, REDISMODULE_READ, 0, &fsl, 0)) return REDISMODULE_ERR; - if (!fsl || fsl->list[fsl->length-1] <= gt) + if (!fsl || fsl->list[fsl->length-1] <= *pgt) return REDISMODULE_ERR; RedisModule_ReplyWithLongLong(ctx, fsl->list[--fsl->length]); @@ -192,10 +192,8 @@ int bpopgt_timeout_callback(RedisModuleCtx *ctx, RedisModuleString **argv, int a } void bpopgt_free_privdata(RedisModuleCtx *ctx, void *privdata) { - /* Nothing to do because privdata is actually a 'long long', - * not a pointer to the heap */ REDISMODULE_NOT_USED(ctx); - REDISMODULE_NOT_USED(privdata); + RedisModule_Free(privdata); } /* FSL.BPOPGT - Block clients until list has an element greater than . @@ -217,9 +215,12 @@ int fsl_bpopgt(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { return REDISMODULE_OK; if (!fsl || fsl->list[fsl->length-1] <= gt) { + /* We use malloc so the tests in blockedonkeys.tcl can check for memory leaks */ + long long *pgt = RedisModule_Alloc(sizeof(long long)); + *pgt = gt; /* Key is empty or has <2 elements, we must block */ RedisModule_BlockClientOnKeys(ctx, bpopgt_reply_callback, bpopgt_timeout_callback, - bpopgt_free_privdata, timeout, &argv[1], 1, (void*)gt); + bpopgt_free_privdata, timeout, &argv[1], 1, pgt); } else { RedisModule_ReplyWithLongLong(ctx, fsl->list[--fsl->length]); } diff --git a/tests/unit/moduleapi/blockonkeys.tcl b/tests/unit/moduleapi/blockonkeys.tcl index cb99ab1c9..b380227e0 100644 --- a/tests/unit/moduleapi/blockonkeys.tcl +++ b/tests/unit/moduleapi/blockonkeys.tcl @@ -45,18 +45,24 @@ start_server {tags {"modules"}} { test {Module client blocked on keys (with metadata): Timeout} { r del k set rd [redis_deferring_client] + $rd client id + set cid [$rd read] r fsl.push k 33 $rd fsl.bpopgt k 35 1 assert_equal {Request timedout} [$rd read] + r client kill id $cid ;# try to smoke-out client-related memory leak } test {Module client blocked on keys (with metadata): Blocked, case 1} { r del k set rd [redis_deferring_client] + $rd client id + set cid [$rd read] r fsl.push k 33 $rd fsl.bpopgt k 33 0 r fsl.push k 34 assert_equal {34} [$rd read] + r client kill id $cid ;# try to smoke-out client-related memory leak } test {Module client blocked on keys (with metadata): Blocked, case 2} { @@ -70,6 +76,35 @@ start_server {tags {"modules"}} { assert_equal {36} [$rd read] } + test {Module client blocked on keys (with metadata): Blocked, CLIENT KILL} { + r del k + set rd [redis_deferring_client] + $rd client id + set cid [$rd read] + $rd fsl.bpopgt k 35 0 + r client kill id $cid ;# try to smoke-out client-related memory leak + } + + test {Module client blocked on keys (with metadata): Blocked, CLIENT UNBLOCK TIMEOUT} { + r del k + set rd [redis_deferring_client] + $rd client id + set cid [$rd read] + $rd fsl.bpopgt k 35 0 + r client unblock $cid timeout ;# try to smoke-out client-related memory leak + assert_equal {Request timedout} [$rd read] + } + + test {Module client blocked on keys (with metadata): Blocked, CLIENT UNBLOCK ERROR} { + r del k + set rd [redis_deferring_client] + $rd client id + set cid [$rd read] + $rd fsl.bpopgt k 35 0 + r client unblock $cid error ;# try to smoke-out client-related memory leak + assert_error "*unblocked*" {$rd read} + } + test {Module client blocked on keys does not wake up on wrong type} { r del k set rd [redis_deferring_client] From 027d609cb0e4bbf79e8b7dc91e11d98c9b572671 Mon Sep 17 00:00:00 2001 From: hayashier Date: Tue, 31 Dec 2019 17:46:48 +0900 Subject: [PATCH 0021/1280] fix typo from fss to rss --- src/object.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/object.c b/src/object.c index 2201a317a..3ae164919 100644 --- a/src/object.c +++ b/src/object.c @@ -1119,13 +1119,13 @@ sds getMemoryDoctorReport(void) { num_reports++; } - /* Allocator fss is higher than 1.1 and 10MB ? */ + /* Allocator rss is higher than 1.1 and 10MB ? */ if (mh->allocator_rss > 1.1 && mh->allocator_rss_bytes > 10<<20) { high_alloc_rss = 1; num_reports++; } - /* Non-Allocator fss is higher than 1.1 and 10MB ? */ + /* Non-Allocator rss is higher than 1.1 and 10MB ? */ if (mh->rss_extra > 1.1 && mh->rss_extra_bytes > 10<<20) { high_proc_rss = 1; num_reports++; From 8aa821fd0a25533cb01208078e3b88703f5f17b8 Mon Sep 17 00:00:00 2001 From: WuYunlong Date: Tue, 31 Dec 2019 18:15:21 +0800 Subject: [PATCH 0022/1280] Fix petential cluster link error. Funcion adjustOpenFilesLimit() has an implicit parameter, which is server.maxclients. This function aims to ajust maximum file descriptor number according to server.maxclients by best effort, which is "bestlimit" could be lower than "maxfiles" but greater than "oldlimit". When we try to increase "maxclients" using CONFIG SET command, we could increase maximum file descriptor number to a bigger value without calling aeResizeSetSize the same time. When later more and more clients connect to server, the allocated fd could be bigger and bigger, and eventually exceeds events size of aeEventLoop.events. When new nodes joins the cluster, new link is created, together with new fd, but when calling aeCreateFileEvent, we did not check the return value. In this case, we have a non-null "link" but the associated fd is not registered. So when we dynamically set "maxclients" we could reach an inconsistency between maximum file descriptor number of the process and server.maxclients. And later could cause cluster link and link fd inconsistency. While setting "maxclients" dynamically, we consider it as failed when resulting "maxclients" is not the same as expected. We try to restore back the maximum file descriptor number when we failed to set "maxclients" to the specified value, so that server.maxclients could act as a guard as before. --- src/config.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/config.c b/src/config.c index 9aa847183..ed19d336b 100644 --- a/src/config.c +++ b/src/config.c @@ -2098,6 +2098,10 @@ static int updateMaxclients(long long val, long long prev, char **err) { static char msg[128]; sprintf(msg, "The operating system is not able to handle the specified number of clients, try with %d", server.maxclients); *err = msg; + if (server.maxclients > prev) { + server.maxclients = prev; + adjustOpenFilesLimit(); + } return 0; } if ((unsigned int) aeGetSetSize(server.el) < From 834f18ab03230281e89188f46003c83325e5010e Mon Sep 17 00:00:00 2001 From: wangyuan21 Date: Tue, 31 Dec 2019 19:53:00 +0800 Subject: [PATCH 0023/1280] free time event when delete eventloop --- src/ae.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/ae.c b/src/ae.c index 2c1dae512..248096e1f 100644 --- a/src/ae.c +++ b/src/ae.c @@ -135,6 +135,13 @@ void aeDeleteEventLoop(aeEventLoop *eventLoop) { aeApiFree(eventLoop); zfree(eventLoop->events); zfree(eventLoop->fired); + /* Free time event. */ + aeTimeEvent *next_te, *te = eventLoop->timeEventHead; + while (te) { + next_te = te->next; + zfree(te); + te = next_te; + } zfree(eventLoop); } From fd67d7192861e568d1761a5764f28aa52bc0b2d0 Mon Sep 17 00:00:00 2001 From: ShooterIT Date: Tue, 31 Dec 2019 21:35:56 +0800 Subject: [PATCH 0024/1280] Rename rdb asynchronously --- src/replication.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/replication.c b/src/replication.c index 68dc77a61..b7e77184a 100644 --- a/src/replication.c +++ b/src/replication.c @@ -31,6 +31,7 @@ #include "server.h" #include "cluster.h" +#include "bio.h" #include #include @@ -1616,14 +1617,20 @@ void readSyncBulkPayload(connection *conn) { killRDBChild(); } + /* Rename rdb like renaming rewrite aof asynchronously. */ + int old_rdb_fd = open(server.rdb_filename,O_RDONLY|O_NONBLOCK); if (rename(server.repl_transfer_tmpfile,server.rdb_filename) == -1) { serverLog(LL_WARNING, "Failed trying to rename the temp DB into %s in " "MASTER <-> REPLICA synchronization: %s", server.rdb_filename, strerror(errno)); cancelReplicationHandshake(); + if (old_rdb_fd != -1) close(old_rdb_fd); return; } + /* Close old rdb asynchronously. */ + if (old_rdb_fd != -1) bioCreateBackgroundJob(BIO_CLOSE_FILE,(void*)(long)old_rdb_fd,NULL,NULL); + if (rdbLoad(server.rdb_filename,&rsi,RDBFLAGS_REPLICATION) != C_OK) { serverLog(LL_WARNING, "Failed trying to load the MASTER synchronization " From e5863f8ff5a792f029d90abf48278add4466aa1b Mon Sep 17 00:00:00 2001 From: John Sully Date: Wed, 1 Jan 2020 10:33:02 -0500 Subject: [PATCH 0025/1280] Add support for incremental build with header files --- .gitignore | 1 + src/Makefile | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 717bf3c7c..8489dbacf 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ .*.swp *.o +*.d *.log dump.rdb redis-benchmark diff --git a/src/Makefile b/src/Makefile index 86a763e31..00b623a4b 100644 --- a/src/Makefile +++ b/src/Makefile @@ -283,14 +283,18 @@ $(REDIS_BENCHMARK_NAME): $(REDIS_BENCHMARK_OBJ) dict-benchmark: dict.c zmalloc.c sds.c siphash.c $(REDIS_CC) $(FINAL_CFLAGS) $^ -D DICT_BENCHMARK_MAIN -o $@ $(FINAL_LIBS) +DEP = $(REDIS_SERVER_OBJ:%.o=%.d) $(REDIS_CLI_OBJ:%.o=%.d) $(REDIS_BENCHMARK_OBJ:%.o=%.d) +-include $(DEP) + # Because the jemalloc.h header is generated as a part of the jemalloc build, # building it should complete before building any other object. Instead of # depending on a single artifact, build all dependencies first. %.o: %.c .make-prerequisites - $(REDIS_CC) -c $< + $(REDIS_CC) -MMD -o $@ -c $< clean: rm -rf $(REDIS_SERVER_NAME) $(REDIS_SENTINEL_NAME) $(REDIS_CLI_NAME) $(REDIS_BENCHMARK_NAME) $(REDIS_CHECK_RDB_NAME) $(REDIS_CHECK_AOF_NAME) *.o *.gcda *.gcno *.gcov redis.info lcov-html Makefile.dep dict-benchmark + rm -f $(DEP) .PHONY: clean From d7691127f78ef50800dc0753a745ba2ebebac545 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 1 Jan 2020 18:10:39 +0100 Subject: [PATCH 0026/1280] Fix active expire division by zero. Likely fix #6723. This is what happens AFAIK: we enter the main loop where we expire stuff until a given percentage of keys is still found to be logically expired. There are however other potential exit conditions. However the "sampled" variable is not always incremented inside the loop, because we may found no valid slot as we scan the hash table, but just NULLs ad dict entries. So when the do/while loop condition is triggered at the end, we do (expired*100/sampled), dividing by zero if we sampled 0 keys. --- src/expire.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/expire.c b/src/expire.c index b4ab9ab18..5aff72ee0 100644 --- a/src/expire.c +++ b/src/expire.c @@ -203,8 +203,10 @@ void activeExpireCycle(int type) { * distribute the time evenly across DBs. */ current_db++; - /* Continue to expire if at the end of the cycle more than 25% - * of the keys were expired. */ + /* Continue to expire if at the end of the cycle there are still + * a big percentage of keys to expire, compared to the number of keys + * we scanned. The percentage, stored in config_cycle_acceptable_stale + * is not fixed, but depends on the Redis configured "expire effort". */ do { unsigned long num, slots; long long now, ttl_sum; @@ -305,8 +307,9 @@ void activeExpireCycle(int type) { } /* We don't repeat the cycle for the current database if there are * an acceptable amount of stale keys (logically expired but yet - * not reclained). */ - } while ((expired*100/sampled) > config_cycle_acceptable_stale); + * not reclaimed). */ + } while (sampled == 0 || + (expired*100/sampled) > config_cycle_acceptable_stale); } elapsed = ustime()-start; From 74a42900631e6120645edfa80a614eb85e0231e6 Mon Sep 17 00:00:00 2001 From: Itamar Haber Date: Sat, 4 Jan 2020 18:33:24 +0200 Subject: [PATCH 0027/1280] Adjusts 'io_threads_num' max to 128 Instead of 512, use the defined max from networking.c --- src/config.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/config.c b/src/config.c index 9aa847183..54d14d38f 100644 --- a/src/config.c +++ b/src/config.c @@ -2192,7 +2192,7 @@ standardConfig configs[] = { /* Integer configs */ createIntConfig("databases", NULL, IMMUTABLE_CONFIG, 1, INT_MAX, server.dbnum, 16, INTEGER_CONFIG, NULL, NULL), createIntConfig("port", NULL, IMMUTABLE_CONFIG, 0, 65535, server.port, 6379, INTEGER_CONFIG, NULL, NULL), /* TCP port. */ - createIntConfig("io-threads", NULL, IMMUTABLE_CONFIG, 1, 512, server.io_threads_num, 1, INTEGER_CONFIG, NULL, NULL), /* Single threaded by default */ + createIntConfig("io-threads", NULL, IMMUTABLE_CONFIG, 1, 128, server.io_threads_num, 1, INTEGER_CONFIG, NULL, NULL), /* Single threaded by default */ createIntConfig("auto-aof-rewrite-percentage", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.aof_rewrite_perc, 100, INTEGER_CONFIG, NULL, NULL), createIntConfig("cluster-replica-validity-factor", "cluster-slave-validity-factor", MODIFIABLE_CONFIG, 0, INT_MAX, server.cluster_slave_validity_factor, 10, INTEGER_CONFIG, NULL, NULL), /* Slave max data age factor. */ createIntConfig("list-max-ziplist-size", NULL, MODIFIABLE_CONFIG, INT_MIN, INT_MAX, server.list_max_ziplist_size, -2, INTEGER_CONFIG, NULL, NULL), From 49bbadb8556fe70edb1e865f4fd71d14bc62a8a2 Mon Sep 17 00:00:00 2001 From: WuYunlong Date: Mon, 6 Jan 2020 19:56:50 +0800 Subject: [PATCH 0028/1280] Fix potential memory leak of rioWriteBulkStreamID(). --- src/aof.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/aof.c b/src/aof.c index 0ef59cfb6..102c39f3f 100644 --- a/src/aof.c +++ b/src/aof.c @@ -1139,7 +1139,10 @@ int rioWriteBulkStreamID(rio *r,streamID *id) { int retval; sds replyid = sdscatfmt(sdsempty(),"%U-%U",id->ms,id->seq); - if ((retval = rioWriteBulkString(r,replyid,sdslen(replyid))) == 0) return 0; + if ((retval = rioWriteBulkString(r,replyid,sdslen(replyid))) == 0) { + sdsfree(replyid); + return 0; + } sdsfree(replyid); return retval; } From e2b8c9280bcf8d033e33dda92cc232af07a7e2b6 Mon Sep 17 00:00:00 2001 From: WuYunlong Date: Tue, 7 Jan 2020 10:28:36 +0800 Subject: [PATCH 0029/1280] Fix potential memory leak of clusterLoadConfig(). --- src/cluster.c | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/src/cluster.c b/src/cluster.c index f603361cd..f9d8ae151 100644 --- a/src/cluster.c +++ b/src/cluster.c @@ -157,7 +157,10 @@ int clusterLoadConfig(char *filename) { } /* Regular config lines have at least eight fields */ - if (argc < 8) goto fmterr; + if (argc < 8) { + sdsfreesplitres(argv,argc); + goto fmterr; + } /* Create this node if it does not exist */ n = clusterLookupNode(argv[0]); @@ -166,7 +169,10 @@ int clusterLoadConfig(char *filename) { clusterAddNode(n); } /* Address and port */ - if ((p = strrchr(argv[1],':')) == NULL) goto fmterr; + if ((p = strrchr(argv[1],':')) == NULL) { + sdsfreesplitres(argv,argc); + goto fmterr; + } *p = '\0'; memcpy(n->ip,argv[1],strlen(argv[1])+1); char *port = p+1; @@ -247,7 +253,10 @@ int clusterLoadConfig(char *filename) { *p = '\0'; direction = p[1]; /* Either '>' or '<' */ slot = atoi(argv[j]+1); - if (slot < 0 || slot >= CLUSTER_SLOTS) goto fmterr; + if (slot < 0 || slot >= CLUSTER_SLOTS) { + sdsfreesplitres(argv,argc); + goto fmterr; + } p += 3; cn = clusterLookupNode(p); if (!cn) { @@ -267,8 +276,14 @@ int clusterLoadConfig(char *filename) { } else { start = stop = atoi(argv[j]); } - if (start < 0 || start >= CLUSTER_SLOTS) goto fmterr; - if (stop < 0 || stop >= CLUSTER_SLOTS) goto fmterr; + if (start < 0 || start >= CLUSTER_SLOTS) { + sdsfreesplitres(argv,argc); + goto fmterr; + } + if (stop < 0 || stop >= CLUSTER_SLOTS) { + sdsfreesplitres(argv,argc); + goto fmterr; + } while(start <= stop) clusterAddSlot(n, start++); } From a127e0cc7dfd78bb1f748e7e19ae00a674741fff Mon Sep 17 00:00:00 2001 From: yz1509 Date: Tue, 7 Jan 2020 10:29:54 +0800 Subject: [PATCH 0030/1280] avoid sentinel changes promoted_slave to be its own replica. --- src/sentinel.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sentinel.c b/src/sentinel.c index 10117252d..10c003d03 100644 --- a/src/sentinel.c +++ b/src/sentinel.c @@ -4308,7 +4308,7 @@ void sentinelFailoverDetectEnd(sentinelRedisInstance *master) { sentinelRedisInstance *slave = dictGetVal(de); int retval; - if (slave->flags & (SRI_RECONF_DONE|SRI_RECONF_SENT)) continue; + if (slave->flags & (SRI_PROMOTED|SRI_RECONF_DONE|SRI_RECONF_SENT)) continue; if (slave->link->disconnected) continue; retval = sentinelSendSlaveOf(slave, From fc9f1b9012dbd1a577e94053324dd8d415a64483 Mon Sep 17 00:00:00 2001 From: WuYunlong Date: Tue, 7 Jan 2020 11:17:52 +0800 Subject: [PATCH 0031/1280] Free allocated sds in pfdebugCommand() to avoid memory leak. --- src/hyperloglog.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/hyperloglog.c b/src/hyperloglog.c index a44d15646..facd99743 100644 --- a/src/hyperloglog.c +++ b/src/hyperloglog.c @@ -1535,6 +1535,7 @@ void pfdebugCommand(client *c) { sds decoded = sdsempty(); if (hdr->encoding != HLL_SPARSE) { + sdsfree(decoded); addReplyError(c,"HLL encoding is not sparse"); return; } From 4a1e1a8c244f955900caad70681ea4a269045b7f Mon Sep 17 00:00:00 2001 From: Leo Murillo Date: Tue, 7 Jan 2020 13:55:26 -0600 Subject: [PATCH 0032/1280] Fix bug on KEYS command where pattern starts with * followed by \x00 (null char). --- src/db.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/db.c b/src/db.c index 32bb35652..ba7be2725 100644 --- a/src/db.c +++ b/src/db.c @@ -602,7 +602,7 @@ void keysCommand(client *c) { void *replylen = addReplyDeferredLen(c); di = dictGetSafeIterator(c->db->dict); - allkeys = (pattern[0] == '*' && pattern[1] == '\0'); + allkeys = (pattern[0] == '*' && plen == 1); while((de = dictNext(di)) != NULL) { sds key = dictGetKey(de); robj *keyobj; From 2a6a01cbf247119d04ee75c1c31418668db62375 Mon Sep 17 00:00:00 2001 From: hwware Date: Tue, 7 Jan 2020 21:09:44 -0500 Subject: [PATCH 0033/1280] typo fix in acl.c --- src/acl.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/acl.c b/src/acl.c index db742c649..1f395bd3f 100644 --- a/src/acl.c +++ b/src/acl.c @@ -183,12 +183,12 @@ int ACLListMatchSds(void *a, void *b) { return sdscmp(a,b) == 0; } -/* Method to free list elements from ACL users password/ptterns lists. */ +/* Method to free list elements from ACL users password/patterns lists. */ void ACLListFreeSds(void *item) { sdsfree(item); } -/* Method to duplicate list elements from ACL users password/ptterns lists. */ +/* Method to duplicate list elements from ACL users password/patterns lists. */ void *ACLListDupSds(void *item) { return sdsdup(item); } From 4573a41249be1e9074d0b4a4c826cefed0dd96bd Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 8 Jan 2020 10:10:11 +0100 Subject: [PATCH 0034/1280] XCLAIM: Create the consumer only on successful claims. Fixes #6744. --- src/t_stream.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/t_stream.c b/src/t_stream.c index 3d46ca0da..ae57202c1 100644 --- a/src/t_stream.c +++ b/src/t_stream.c @@ -2246,7 +2246,7 @@ void xclaimCommand(client *c) { } /* Do the actual claiming. */ - streamConsumer *consumer = streamLookupConsumer(group,c->argv[3]->ptr,1); + streamConsumer *consumer = NULL; void *arraylenptr = addReplyDeferredLen(c); size_t arraylen = 0; for (int j = 5; j <= last_id_arg; j++) { @@ -2298,9 +2298,11 @@ void xclaimCommand(client *c) { if (nack->consumer) raxRemove(nack->consumer->pel,buf,sizeof(buf),NULL); /* Update the consumer and idle time. */ + if (consumer == NULL) + consumer = streamLookupConsumer(group,c->argv[3]->ptr,1); nack->consumer = consumer; nack->delivery_time = deliverytime; - /* Set the delivery attempts counter if given, otherwise + /* Set the delivery attempts counter if given, otherwise * autoincrement unless JUSTID option provided */ if (retrycount >= 0) { nack->delivery_count = retrycount; From c8decc87e80094739431ba4b9a2bca5baf8ea38e Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 10 Jan 2020 12:22:16 +0100 Subject: [PATCH 0035/1280] Git ignore: ignore more files. --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 8489dbacf..de626d61b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,7 @@ .*.swp *.o +*.xo +*.so *.d *.log dump.rdb From b2754167467f6246f244d4c7f259b36314f403a1 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 10 Jan 2020 13:02:45 +0100 Subject: [PATCH 0036/1280] Free fakeclient argv on AOF error. We exit later, so no bug fixed, but it is more correct. See #6054, thanks to @ShooterIT for finding the issue. --- src/aof.c | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/aof.c b/src/aof.c index 0ef59cfb6..9c2fa838b 100644 --- a/src/aof.c +++ b/src/aof.c @@ -781,18 +781,24 @@ int loadAppendOnlyFile(char *filename) { argc = atoi(buf+1); if (argc < 1) goto fmterr; + /* Load the next command in the AOF as our fake client + * argv. */ argv = zmalloc(sizeof(robj*)*argc); fakeClient->argc = argc; fakeClient->argv = argv; for (j = 0; j < argc; j++) { - if (fgets(buf,sizeof(buf),fp) == NULL) { + /* Parse the argument len. */ + if (fgets(buf,sizeof(buf),fp) == NULL || + buf[0] != '$') + { fakeClient->argc = j; /* Free up to j-1. */ freeFakeClientArgv(fakeClient); goto readerr; } - if (buf[0] != '$') goto fmterr; len = strtol(buf+1,NULL,10); + + /* Read it into a string object. */ argsds = sdsnewlen(SDS_NOINIT,len); if (len && fread(argsds,len,1,fp) == 0) { sdsfree(argsds); @@ -801,10 +807,12 @@ int loadAppendOnlyFile(char *filename) { goto readerr; } argv[j] = createObject(OBJ_STRING,argsds); + + /* Discard CRLF. */ if (fread(buf,2,1,fp) == 0) { fakeClient->argc = j+1; /* Free up to j. */ freeFakeClientArgv(fakeClient); - goto readerr; /* discard CRLF */ + goto readerr; } } From 9ea3bfdb8b7e178f403febabc36db87828f1884e Mon Sep 17 00:00:00 2001 From: Vasyl Melnychuk Date: Fri, 10 Jan 2020 23:34:15 +0200 Subject: [PATCH 0037/1280] Make error when submitting command in incorrect context more explicit So error message `ERR only (P)SUBSCRIBE / (P)UNSUBSCRIBE / PING / QUIT allowed in this context` will become `ERR 'get' command submitted, but only (P)SUBSCRIBE / (P)UNSUBSCRIBE / PING / QUIT allowed in this context` --- src/server.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/server.c b/src/server.c index 5845a5485..3f64aaec9 100644 --- a/src/server.c +++ b/src/server.c @@ -3498,7 +3498,10 @@ int processCommand(client *c) { c->cmd->proc != unsubscribeCommand && c->cmd->proc != psubscribeCommand && c->cmd->proc != punsubscribeCommand) { - addReplyError(c,"only (P)SUBSCRIBE / (P)UNSUBSCRIBE / PING / QUIT allowed in this context"); + addReplyErrorFormat(c, + "'%s' command submitted, but only (P)SUBSCRIBE / " + "(P)UNSUBSCRIBE / PING / QUIT allowed in this context", + c->cmd->name); return C_OK; } From 9d6f439760b066f3897b1a584d7753a6925ad551 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 13 Jan 2020 12:50:26 +0100 Subject: [PATCH 0038/1280] A few comments about main thread serving I/O as well. Related to #6110. --- src/networking.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/networking.c b/src/networking.c index f1a6b9910..96ab8e592 100644 --- a/src/networking.c +++ b/src/networking.c @@ -2656,6 +2656,10 @@ pthread_mutex_t io_threads_mutex[IO_THREADS_MAX_NUM]; _Atomic unsigned long io_threads_pending[IO_THREADS_MAX_NUM]; int io_threads_active; /* Are the threads currently spinning waiting I/O? */ int io_threads_op; /* IO_THREADS_OP_WRITE or IO_THREADS_OP_READ. */ + +/* This is the list of clients each thread will serve when threaded I/O is + * used. We spawn N threads, and the N+1 list is used for the clients that + * are processed by the main thread itself (this is why ther is "+1"). */ list *io_threads_list[IO_THREADS_MAX_NUM+1]; void *IOThreadMain(void *myid) { @@ -2729,7 +2733,7 @@ void initThreadedIO(void) { } io_threads[i] = tid; } - io_threads_list[server.io_threads_num] = listCreate(); + io_threads_list[server.io_threads_num] = listCreate(); /* For main thread */ } void startThreadedIO(void) { @@ -2814,6 +2818,7 @@ int handleClientsWithPendingWritesUsingThreads(void) { io_threads_pending[j] = count; } + /* Also use the main thread to process a slide of clients. */ listRewind(io_threads_list[server.io_threads_num],&li); while((ln = listNext(&li))) { client *c = listNodeValue(ln); @@ -2898,6 +2903,7 @@ int handleClientsWithPendingReadsUsingThreads(void) { io_threads_pending[j] = count; } + /* Also use the main thread to process a slide of clients. */ listRewind(io_threads_list[server.io_threads_num],&li); while((ln = listNext(&li))) { client *c = listNodeValue(ln); From ff9e42365f1296d5ef72a2e0d3b51651ee7fce92 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 13 Jan 2020 12:54:39 +0100 Subject: [PATCH 0039/1280] Port PR #6110 to new connection object code. --- src/networking.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/networking.c b/src/networking.c index 96ab8e592..73dc4afca 100644 --- a/src/networking.c +++ b/src/networking.c @@ -2822,7 +2822,7 @@ int handleClientsWithPendingWritesUsingThreads(void) { listRewind(io_threads_list[server.io_threads_num],&li); while((ln = listNext(&li))) { client *c = listNodeValue(ln); - writeToClient(c->fd,c,0); + writeToClient(c,0); } listEmpty(io_threads_list[server.io_threads_num]); @@ -2907,7 +2907,7 @@ int handleClientsWithPendingReadsUsingThreads(void) { listRewind(io_threads_list[server.io_threads_num],&li); while((ln = listNext(&li))) { client *c = listNodeValue(ln); - readQueryFromClient(NULL,c->fd,c,0); + readQueryFromClient(c->conn); } listEmpty(io_threads_list[server.io_threads_num]); From 5ac23a276541ac0b76e207f84b01c82a6e3e6f4a Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 13 Jan 2020 13:16:13 +0100 Subject: [PATCH 0040/1280] Jump to right label on AOF parsing error. Related to #6054. --- src/aof.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/aof.c b/src/aof.c index 9c2fa838b..63b34b43f 100644 --- a/src/aof.c +++ b/src/aof.c @@ -789,12 +789,14 @@ int loadAppendOnlyFile(char *filename) { for (j = 0; j < argc; j++) { /* Parse the argument len. */ - if (fgets(buf,sizeof(buf),fp) == NULL || - buf[0] != '$') - { + char *readres = fgets(buf,sizeof(buf),fp); + if (readres == NULL || buf[0] != '$') { fakeClient->argc = j; /* Free up to j-1. */ freeFakeClientArgv(fakeClient); - goto readerr; + if (readres == NULL) + goto readerr; + else + goto fmterr; } len = strtol(buf+1,NULL,10); From 5c5f25a9f9664116abd76f1973c07c1425a1626f Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 13 Jan 2020 13:25:37 +0100 Subject: [PATCH 0041/1280] Simplify #6379 changes. --- src/aof.c | 5 +---- src/cluster.c | 8 +++----- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/src/aof.c b/src/aof.c index efdd68efa..9eeb3f1e2 100644 --- a/src/aof.c +++ b/src/aof.c @@ -1149,10 +1149,7 @@ int rioWriteBulkStreamID(rio *r,streamID *id) { int retval; sds replyid = sdscatfmt(sdsempty(),"%U-%U",id->ms,id->seq); - if ((retval = rioWriteBulkString(r,replyid,sdslen(replyid))) == 0) { - sdsfree(replyid); - return 0; - } + retval = rioWriteBulkString(r,replyid,sdslen(replyid)); sdsfree(replyid); return retval; } diff --git a/src/cluster.c b/src/cluster.c index f9d8ae151..c05e46f76 100644 --- a/src/cluster.c +++ b/src/cluster.c @@ -276,11 +276,9 @@ int clusterLoadConfig(char *filename) { } else { start = stop = atoi(argv[j]); } - if (start < 0 || start >= CLUSTER_SLOTS) { - sdsfreesplitres(argv,argc); - goto fmterr; - } - if (stop < 0 || stop >= CLUSTER_SLOTS) { + if (start < 0 || start >= CLUSTER_SLOTS || + stop < 0 || stop >= CLUSTER_SLOTS) + { sdsfreesplitres(argv,argc); goto fmterr; } From aca010264caec333353b2df2ee3f1a466fea4bcd Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 13 Jan 2020 18:53:12 +0100 Subject: [PATCH 0042/1280] Setting N I/O threads should mean N-1 additional + 1 main thread. --- src/networking.c | 47 +++++++++++++++++++++++++---------------------- 1 file changed, 25 insertions(+), 22 deletions(-) diff --git a/src/networking.c b/src/networking.c index 73dc4afca..a2e454d4b 100644 --- a/src/networking.c +++ b/src/networking.c @@ -2658,9 +2658,9 @@ int io_threads_active; /* Are the threads currently spinning waiting I/O? */ int io_threads_op; /* IO_THREADS_OP_WRITE or IO_THREADS_OP_READ. */ /* This is the list of clients each thread will serve when threaded I/O is - * used. We spawn N threads, and the N+1 list is used for the clients that - * are processed by the main thread itself (this is why ther is "+1"). */ -list *io_threads_list[IO_THREADS_MAX_NUM+1]; + * used. We spawn io_threads_num-1 threads, since one is the main thread + * itself. */ +list *io_threads_list[IO_THREADS_MAX_NUM]; void *IOThreadMain(void *myid) { /* The ID is the thread number (from 0 to server.iothreads_num-1), and is @@ -2720,12 +2720,16 @@ void initThreadedIO(void) { exit(1); } - /* Spawn the I/O threads. */ + /* Spawn and initialize the I/O threads. */ for (int i = 0; i < server.io_threads_num; i++) { + /* Things we do for all the threads including the main thread. */ + io_threads_list[i] = listCreate(); + if (i == 0) continue; /* Thread 0 is the main thread. */ + + /* Things we do only for the additional threads. */ pthread_t tid; pthread_mutex_init(&io_threads_mutex[i],NULL); io_threads_pending[i] = 0; - io_threads_list[i] = listCreate(); pthread_mutex_lock(&io_threads_mutex[i]); /* Thread will be stopped. */ if (pthread_create(&tid,NULL,IOThreadMain,(void*)(long)i) != 0) { serverLog(LL_WARNING,"Fatal: Can't initialize IO thread."); @@ -2733,14 +2737,13 @@ void initThreadedIO(void) { } io_threads[i] = tid; } - io_threads_list[server.io_threads_num] = listCreate(); /* For main thread */ } void startThreadedIO(void) { if (tio_debug) { printf("S"); fflush(stdout); } if (tio_debug) printf("--- STARTING THREADED IO ---\n"); serverAssert(io_threads_active == 0); - for (int j = 0; j < server.io_threads_num; j++) + for (int j = 1; j < server.io_threads_num; j++) pthread_mutex_unlock(&io_threads_mutex[j]); io_threads_active = 1; } @@ -2754,7 +2757,7 @@ void stopThreadedIO(void) { (int) listLength(server.clients_pending_read), (int) listLength(server.clients_pending_write)); serverAssert(io_threads_active == 1); - for (int j = 0; j < server.io_threads_num; j++) + for (int j = 1; j < server.io_threads_num; j++) pthread_mutex_lock(&io_threads_mutex[j]); io_threads_active = 0; } @@ -2805,7 +2808,7 @@ int handleClientsWithPendingWritesUsingThreads(void) { while((ln = listNext(&li))) { client *c = listNodeValue(ln); c->flags &= ~CLIENT_PENDING_WRITE; - int target_id = item_id % (server.io_threads_num+1); + int target_id = item_id % server.io_threads_num; listAddNodeTail(io_threads_list[target_id],c); item_id++; } @@ -2813,23 +2816,23 @@ int handleClientsWithPendingWritesUsingThreads(void) { /* Give the start condition to the waiting threads, by setting the * start condition atomic var. */ io_threads_op = IO_THREADS_OP_WRITE; - for (int j = 0; j < server.io_threads_num; j++) { + for (int j = 1; j < server.io_threads_num; j++) { int count = listLength(io_threads_list[j]); io_threads_pending[j] = count; } - /* Also use the main thread to process a slide of clients. */ - listRewind(io_threads_list[server.io_threads_num],&li); + /* Also use the main thread to process a slice of clients. */ + listRewind(io_threads_list[0],&li); while((ln = listNext(&li))) { client *c = listNodeValue(ln); writeToClient(c,0); } - listEmpty(io_threads_list[server.io_threads_num]); + listEmpty(io_threads_list[0]); - /* Wait for all threads to end their work. */ + /* Wait for all the other threads to end their work. */ while(1) { unsigned long pending = 0; - for (int j = 0; j < server.io_threads_num; j++) + for (int j = 1; j < server.io_threads_num; j++) pending += io_threads_pending[j]; if (pending == 0) break; } @@ -2890,7 +2893,7 @@ int handleClientsWithPendingReadsUsingThreads(void) { int item_id = 0; while((ln = listNext(&li))) { client *c = listNodeValue(ln); - int target_id = item_id % (server.io_threads_num+1); + int target_id = item_id % server.io_threads_num; listAddNodeTail(io_threads_list[target_id],c); item_id++; } @@ -2898,23 +2901,23 @@ int handleClientsWithPendingReadsUsingThreads(void) { /* Give the start condition to the waiting threads, by setting the * start condition atomic var. */ io_threads_op = IO_THREADS_OP_READ; - for (int j = 0; j < server.io_threads_num; j++) { + for (int j = 1; j < server.io_threads_num; j++) { int count = listLength(io_threads_list[j]); io_threads_pending[j] = count; } - /* Also use the main thread to process a slide of clients. */ - listRewind(io_threads_list[server.io_threads_num],&li); + /* Also use the main thread to process a slice of clients. */ + listRewind(io_threads_list[0],&li); while((ln = listNext(&li))) { client *c = listNodeValue(ln); readQueryFromClient(c->conn); } - listEmpty(io_threads_list[server.io_threads_num]); + listEmpty(io_threads_list[0]); - /* Wait for all threads to end their work. */ + /* Wait for all the other threads to end their work. */ while(1) { unsigned long pending = 0; - for (int j = 0; j < server.io_threads_num; j++) + for (int j = 1; j < server.io_threads_num; j++) pending += io_threads_pending[j]; if (pending == 0) break; } From 4b1de0575751c66a62288b1e2fcba54393da1c2a Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 13 Jan 2020 19:10:42 +0100 Subject: [PATCH 0043/1280] Document I/O threads in redis.conf. --- redis.conf | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/redis.conf b/redis.conf index 5f9547c1d..07005cffe 100644 --- a/redis.conf +++ b/redis.conf @@ -938,6 +938,52 @@ lazyfree-lazy-expire no lazyfree-lazy-server-del no replica-lazy-flush no +################################ THREADED I/O ################################# + +# Redis is mostly single threaded, however there are certain threaded +# operations such as UNLINK, slow I/O accesses and other things that are +# performed on side threads. +# +# Now it is also possible to handle Redis clients socket reads and writes +# in different I/O threads. Since especially writing is so slow, normally +# Redis users use pipelining in order to speedup the Redis performances per +# core, and spawn multiple instances in order to scale more. Using I/O +# threads it is possible to easily speedup two times Redis without resorting +# to pipelining nor sharding of the instance. +# +# By default threading is disabled, we suggest enabling it only in machines +# that have at least 4 or more cores, leaving at least one spare core. +# Using more than 8 threads is unlikely to help much. We also recommend using +# threaded I/O only if you actually have performance problems, with Redis +# instances being able to use a quite big percentage of CPU time, otherwise +# there is no point in using this feature. +# +# So for instance if you have a four cores boxes, try to use 2 or 3 I/O +# threads, if you have a 8 cores, try to use 6 threads. In order to +# enable I/O threads use the following configuration directive: +# +# io-threads 4 +# +# Setting io-threads to 1 will just use the main thread as usually. +# When I/O threads are enabled, we only use threads for writes, that is +# to thread the write(2) syscall and transfer the client buffers to the +# socket. However it is also possible to enable threading of reads and +# protocol parsing using the following configuration directive, by setting +# it to yes: +# +# io-threads-do-reads no +# +# Usually threading reads doesn't help much. +# +# NOTE 1: This configuration directive cannot be changed at runtime via +# CONFIG SET. Aso this feature currently does not work when SSL is +# enabled. +# +# NOTE 2: If you want to test the Redis speedup using redis-benchmark, make +# sure you also run the benchmark itself in threaded mode, using the +# --threads option to match the number of Redis theads, otherwise you'll not +# be able to notice the improvements. + ############################## APPEND ONLY MODE ############################### # By default Redis asynchronously dumps the dataset on disk. This mode is From 5b1fc9ce6e370d32882db75d0ae49f8cdd940e79 Mon Sep 17 00:00:00 2001 From: Ponnuvel Palaniyappan Date: Tue, 14 Jan 2020 07:38:57 +0000 Subject: [PATCH 0044/1280] Fix typos in README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 3442659e6..9d7c3fbcb 100644 --- a/README.md +++ b/README.md @@ -318,7 +318,7 @@ There are two special functions called periodically by the event loop: Inside server.c you can find code that handles other vital things of the Redis server: * `call()` is used in order to call a given command in the context of a given client. -* `activeExpireCycle()` handles eviciton of keys with a time to live set via the `EXPIRE` command. +* `activeExpireCycle()` handles eviction of keys with a time to live set via the `EXPIRE` command. * `freeMemoryIfNeeded()` is called when a new write command should be performed but Redis is out of memory according to the `maxmemory` directive. * The global variable `redisCommandTable` defines all the Redis commands, specifying the name of the command, the function implementing the command, the number of arguments required, and other properties of each command. @@ -411,7 +411,7 @@ Other C files * `sds.c` is the Redis string library, check http://github.com/antirez/sds for more information. * `anet.c` is a library to use POSIX networking in a simpler way compared to the raw interface exposed by the kernel. * `dict.c` is an implementation of a non-blocking hash table which rehashes incrementally. -* `scripting.c` implements Lua scripting. It is completely self contained from the rest of the Redis implementation and is simple enough to understand if you are familar with the Lua API. +* `scripting.c` implements Lua scripting. It is completely self contained from the rest of the Redis implementation and is simple enough to understand if you are familiar with the Lua API. * `cluster.c` implements the Redis Cluster. Probably a good read only after being very familiar with the rest of the Redis code base. If you want to read `cluster.c` make sure to read the [Redis Cluster specification][3]. [3]: http://redis.io/topics/cluster-spec From 9df0a501eeefecfd8754caf90e200efbfc8c1e71 Mon Sep 17 00:00:00 2001 From: Ponnuvel Palaniyappan Date: Tue, 14 Jan 2020 08:10:39 +0000 Subject: [PATCH 0045/1280] Fix a potential overflow with strncpy --- src/config.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/config.c b/src/config.c index 0526de84d..e4b1bf869 100644 --- a/src/config.c +++ b/src/config.c @@ -108,12 +108,12 @@ clientBufferLimitsConfig clientBufferLimitsDefaults[CLIENT_TYPE_OBUF_COUNT] = { /* Generic config infrastructure function pointers * int is_valid_fn(val, err) * Return 1 when val is valid, and 0 when invalid. - * Optionslly set err to a static error string. + * Optionally set err to a static error string. * int update_fn(val, prev, err) * This function is called only for CONFIG SET command (not at config file parsing) * It is called after the actual config is applied, * Return 1 for success, and 0 for failure. - * Optionslly set err to a static error string. + * Optionally set err to a static error string. * On failure the config change will be reverted. */ @@ -720,7 +720,7 @@ void configSetCommand(client *c) { * config_set_memory_field(name,var) */ } config_set_memory_field( "client-query-buffer-limit",server.client_max_querybuf_len) { - /* Everyhing else is an error... */ + /* Everything else is an error... */ } config_set_else { addReplyErrorFormat(c,"Unsupported CONFIG parameter: %s", (char*)c->argv[2]->ptr); @@ -1699,9 +1699,9 @@ static int configEnumLoad(typeData data, sds *argv, int argc, char **err) { enumerr[sdslen(enumerr) - 2] = '\0'; - /* Make sure we don't overrun the fixed buffer */ - enumerr[LOADBUF_SIZE - 1] = '\0'; strncpy(loadbuf, enumerr, LOADBUF_SIZE); + /* strncpy does not if null terminate if source string length is >= destination buffer. */ + loadbuf[LOADBUF_SIZE - 1] = '\0'; sdsfree(enumerr); *err = loadbuf; From c0bdf0eb732f9d3054dd1b43cf1aa54e8e4bf026 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 15 Jan 2020 17:55:24 +0100 Subject: [PATCH 0046/1280] Change error message for #6775. --- src/server.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/server.c b/src/server.c index 3f64aaec9..2b226f568 100644 --- a/src/server.c +++ b/src/server.c @@ -3499,8 +3499,8 @@ int processCommand(client *c) { c->cmd->proc != psubscribeCommand && c->cmd->proc != punsubscribeCommand) { addReplyErrorFormat(c, - "'%s' command submitted, but only (P)SUBSCRIBE / " - "(P)UNSUBSCRIBE / PING / QUIT allowed in this context", + "Can't execute '%s': only (P)SUBSCRIBE / " + "(P)UNSUBSCRIBE / PING / QUIT are allowed in this context", c->cmd->name); return C_OK; } From 89f5aaa4d7390c2ef3e5b453b6e38e3084bb8fe9 Mon Sep 17 00:00:00 2001 From: hwware Date: Thu, 16 Jan 2020 17:33:23 -0500 Subject: [PATCH 0047/1280] fix potentical memory leaks --- src/debug.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/debug.c b/src/debug.c index a2d37337d..b441fa8f8 100644 --- a/src/debug.c +++ b/src/debug.c @@ -683,9 +683,12 @@ NULL sds stats = sdsempty(); char buf[4096]; - if (getLongFromObjectOrReply(c, c->argv[2], &dbid, NULL) != C_OK) + if (getLongFromObjectOrReply(c, c->argv[2], &dbid, NULL) != C_OK){ + sdsfree(stats); return; + } if (dbid < 0 || dbid >= server.dbnum) { + sdsfree(stats); addReplyError(c,"Out of range database"); return; } From 65cffe587c3b733c1941c102ba12f84dc89a4e19 Mon Sep 17 00:00:00 2001 From: hwware Date: Thu, 16 Jan 2020 17:35:26 -0500 Subject: [PATCH 0048/1280] format fix --- src/debug.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/debug.c b/src/debug.c index b441fa8f8..5ffbab7b4 100644 --- a/src/debug.c +++ b/src/debug.c @@ -683,7 +683,7 @@ NULL sds stats = sdsempty(); char buf[4096]; - if (getLongFromObjectOrReply(c, c->argv[2], &dbid, NULL) != C_OK){ + if (getLongFromObjectOrReply(c, c->argv[2], &dbid, NULL) != C_OK) { sdsfree(stats); return; } From 90589504cc6b34eca272488152e3dc56c061b3d7 Mon Sep 17 00:00:00 2001 From: srzhao Date: Fri, 17 Jan 2020 11:46:19 +0800 Subject: [PATCH 0049/1280] fix impl of aof-child whitelist SIGUSR1 feature. --- src/aof.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/aof.c b/src/aof.c index 9eeb3f1e2..ce1a440f8 100644 --- a/src/aof.c +++ b/src/aof.c @@ -1797,14 +1797,15 @@ void backgroundRewriteDoneHandler(int exitcode, int bysignal) { serverLog(LL_VERBOSE, "Background AOF rewrite signal handler took %lldus", ustime()-now); } else if (!bysignal && exitcode != 0) { + server.aof_lastbgrewrite_status = C_ERR; + + serverLog(LL_WARNING, + "Background AOF rewrite terminated with error"); + } else { /* SIGUSR1 is whitelisted, so we have a way to kill a child without * tirggering an error condition. */ if (bysignal != SIGUSR1) server.aof_lastbgrewrite_status = C_ERR; - serverLog(LL_WARNING, - "Background AOF rewrite terminated with error"); - } else { - server.aof_lastbgrewrite_status = C_ERR; serverLog(LL_WARNING, "Background AOF rewrite terminated by signal %d", bysignal); From f6621280c94dc58dde2240e9ffd7b7cdb2fddcc0 Mon Sep 17 00:00:00 2001 From: srzhao Date: Mon, 20 Jan 2020 21:17:02 +0800 Subject: [PATCH 0050/1280] Check OOM at script start to get stable lua OOM state. Checking OOM by `getMaxMemoryState` inside script might get different result with `freeMemoryIfNeededAndSafe` at script start, because lua stack and arguments also consume memory. This leads to memory `borderline` when memory grows near server.maxmemory: - `freeMemoryIfNeededAndSafe` at script start detects no OOM, no memory freed - `getMaxMemoryState` inside script detects OOM, script aborted We solve this 'borderline' issue by saving OOM state at script start to get stable lua OOM state. related to issue #6565 and #5250. --- src/scripting.c | 7 +++---- src/server.c | 7 +++++++ src/server.h | 1 + 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/src/scripting.c b/src/scripting.c index 9282b7fd9..5ed24ffad 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -655,12 +655,11 @@ int luaRedisGenericCommand(lua_State *lua, int raise_error) { !server.loading && /* Don't care about mem if loading. */ !server.masterhost && /* Slave must execute the script. */ server.lua_write_dirty == 0 && /* Script had no side effects so far. */ + server.lua_oom && /* Detected OOM when script start. */ (cmd->flags & CMD_DENYOOM)) { - if (getMaxmemoryState(NULL,NULL,NULL,NULL) != C_OK) { - luaPushError(lua, shared.oomerr->ptr); - goto cleanup; - } + luaPushError(lua, shared.oomerr->ptr); + goto cleanup; } if (cmd->flags & CMD_RANDOM) server.lua_random_dirty = 1; diff --git a/src/server.c b/src/server.c index 2b226f568..21d6aab27 100644 --- a/src/server.c +++ b/src/server.c @@ -3442,6 +3442,13 @@ int processCommand(client *c) { addReply(c, shared.oomerr); return C_OK; } + + /* Save out_of_memory result at script start, otherwise if we check OOM + * untill first write within script, memory used by lua stack and + * arguments might interfere. */ + if (c->cmd->proc == evalCommand || c->cmd->proc == evalShaCommand) { + server.lua_oom = out_of_memory; + } } /* Make sure to use a reasonable amount of memory for client side diff --git a/src/server.h b/src/server.h index 8e354c03d..b440beba3 100644 --- a/src/server.h +++ b/src/server.h @@ -1376,6 +1376,7 @@ struct redisServer { execution. */ int lua_kill; /* Kill the script if true. */ int lua_always_replicate_commands; /* Default replication type. */ + int lua_oom; /* OOM detected when script start? */ /* Lazy free */ int lazyfree_lazy_eviction; int lazyfree_lazy_expire; From 7bd9afe7802eb232cd9d1e4ccdefa23214142ad6 Mon Sep 17 00:00:00 2001 From: qetu3790 Date: Thu, 23 Jan 2020 17:18:07 +0800 Subject: [PATCH 0051/1280] Fix not used constant in lru_test_mode. LRU_CYCLE_PERIOD is defined,but not used. --- src/redis-cli.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/redis-cli.c b/src/redis-cli.c index 065c389c6..36ceb16e8 100644 --- a/src/redis-cli.c +++ b/src/redis-cli.c @@ -7735,7 +7735,7 @@ static void LRUTestMode(void) { * to fill the target instance easily. */ start_cycle = mstime(); long long hits = 0, misses = 0; - while(mstime() - start_cycle < 1000) { + while(mstime() - start_cycle < LRU_CYCLE_PERIOD) { /* Write cycle. */ for (j = 0; j < LRU_CYCLE_PIPELINE_SIZE; j++) { char val[6]; From 800425a319870cb69695a1748953ca66a1fef3e9 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 27 Jan 2020 18:37:52 +0100 Subject: [PATCH 0052/1280] ACL LOG: data structures and initial functions. --- src/acl.c | 51 ++++++++++++++++++++++++++++++++++++++++++++++++- src/multi.c | 2 +- src/scripting.c | 2 +- src/server.c | 2 +- src/server.h | 2 +- 5 files changed, 54 insertions(+), 5 deletions(-) diff --git a/src/acl.c b/src/acl.c index 1f395bd3f..4391382a6 100644 --- a/src/acl.c +++ b/src/acl.c @@ -49,6 +49,8 @@ list *UsersToLoad; /* This is a list of users found in the configuration file array of SDS pointers: the first is the user name, all the remaining pointers are ACL rules in the same format as ACLSetUser(). */ +list *ACLLog; /* Our security log, the user is able to inspect that + using the ACL LOG command .*/ struct ACLCategoryItem { const char *name; @@ -920,6 +922,7 @@ void ACLInitDefaultUser(void) { void ACLInit(void) { Users = raxNew(); UsersToLoad = listCreate(); + ACLLog = listCreate(); ACLInitDefaultUser(); } @@ -1034,7 +1037,7 @@ user *ACLGetUserByName(const char *name, size_t namelen) { * command cannot be executed because the user is not allowed to run such * command, the second if the command is denied because the user is trying * to access keys that are not among the specified patterns. */ -int ACLCheckCommandPerm(client *c) { +int ACLCheckCommandPerm(client *c, int *keyidxptr) { user *u = c->user; uint64_t id = c->cmd->id; @@ -1094,6 +1097,7 @@ int ACLCheckCommandPerm(client *c) { } } if (!match) { + if (keyidxptr) *keyidxptr = keyidx[j]; getKeysFreeResult(keyidx); return ACL_DENIED_KEY; } @@ -1454,6 +1458,51 @@ void ACLLoadUsersAtStartup(void) { } } +/* ============================================================================= + * ACL log + * ==========================================================================*/ + +#define ACL_LOG_CTX_TOPLEVEL 0 +#define ACL_LOG_CTX_LUA 1 +#define ACL_LOG_CTX_MULTI 2 + +/* This structure defines an entry inside the ACL log. */ +typedef struct aclLogEntry { + uint64_t count; /* Number of times this happened recently. */ + int reason; /* Reason for denying the command. ACL_DENIED_*. */ + int context; /* Toplevel, Lua or MULTI/EXEC? ACL_LOG_CTX_*. */ + sds object; /* The key name or command name. */ + sds username; /* User the client is authenticated with. */ + mstime_t ctime; /* Milliseconds time of last update to this entry. */ + sds cinfo; /* Client info (last client if updated). */ +} aclLogEntry; + +void addACLLogEntry(client *c, int reason, int keypos) { + /* Create a new entry. */ + struct aclLogEntry *le = zmalloc(sizeof(*le)); + le->count = 1; + le->object = (reason == ACL_DENIED_CMD) ? sdsnew(c->cmd->name) : + sdsdup(c->argv[keypos]->ptr); + le->username = sdsdup(c->user->name); + le->ctime = mstime(); + + client *realclient = c; + if (realclient->flags & CLIENT_LUA) realclient = server.lua_caller; + + le->cinfo = catClientInfoString(sdsempty(),realclient); + if (c->flags & CLIENT_MULTI) { + le->context = ACL_LOG_CTX_MULTI; + } else if (c->flags & CLIENT_LUA) { + le->context = ACL_LOG_CTX_LUA; + } else { + le->context = ACL_LOG_CTX_TOPLEVEL; + } + + /* Add it to our list of entires. We'll have to trim the list + * to its maximum size. */ + listAddNodeHead(ACLLog, le); +} + /* ============================================================================= * ACL related commands * ==========================================================================*/ diff --git a/src/multi.c b/src/multi.c index df11225bd..640149870 100644 --- a/src/multi.c +++ b/src/multi.c @@ -177,7 +177,7 @@ void execCommand(client *c) { must_propagate = 1; } - int acl_retval = ACLCheckCommandPerm(c); + int acl_retval = ACLCheckCommandPerm(c,NULL); if (acl_retval != ACL_OK) { addReplyErrorFormat(c, "-NOPERM ACLs rules changed between the moment the " diff --git a/src/scripting.c b/src/scripting.c index 9282b7fd9..0e47ebcb0 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -606,7 +606,7 @@ int luaRedisGenericCommand(lua_State *lua, int raise_error) { } /* Check the ACLs. */ - int acl_retval = ACLCheckCommandPerm(c); + int acl_retval = ACLCheckCommandPerm(c,NULL); if (acl_retval != ACL_OK) { if (acl_retval == ACL_DENIED_CMD) luaPushError(lua, "The user executing the script can't run this " diff --git a/src/server.c b/src/server.c index 2b226f568..b5e27e238 100644 --- a/src/server.c +++ b/src/server.c @@ -3377,7 +3377,7 @@ int processCommand(client *c) { /* Check if the user can run this command according to the current * ACLs. */ - int acl_retval = ACLCheckCommandPerm(c); + int acl_retval = ACLCheckCommandPerm(c,NULL); if (acl_retval != ACL_OK) { flagTransaction(c); if (acl_retval == ACL_DENIED_CMD) diff --git a/src/server.h b/src/server.h index 8e354c03d..6fd0ffc5b 100644 --- a/src/server.h +++ b/src/server.h @@ -1824,7 +1824,7 @@ int ACLCheckUserCredentials(robj *username, robj *password); int ACLAuthenticateUser(client *c, robj *username, robj *password); unsigned long ACLGetCommandID(const char *cmdname); user *ACLGetUserByName(const char *name, size_t namelen); -int ACLCheckCommandPerm(client *c); +int ACLCheckCommandPerm(client *c, int *keyidxptr); int ACLSetUser(user *u, const char *op, ssize_t oplen); sds ACLDefaultUserFirstPassword(void); uint64_t ACLGetCommandCategoryFlagByName(const char *name); From 99a5b98bf041c407e1f9812ccdf7303d2390a478 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 28 Jan 2020 17:30:50 +0100 Subject: [PATCH 0053/1280] ACL LOG: implement ACL LOG subcommadn skeleton. --- src/acl.c | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/src/acl.c b/src/acl.c index 4391382a6..a166469d0 100644 --- a/src/acl.c +++ b/src/acl.c @@ -1477,6 +1477,11 @@ typedef struct aclLogEntry { sds cinfo; /* Client info (last client if updated). */ } aclLogEntry; +/* Adds a new entry in the ACL log, making sure to delete the old entry + * if we reach the maximum length allowed for the log. This function attempts + * to find similar entries in the current log in order to bump the counter of + * the log entry instead of creating many entries for very similar ACL + * rules issues. */ void addACLLogEntry(client *c, int reason, int keypos) { /* Create a new entry. */ struct aclLogEntry *le = zmalloc(sizeof(*le)); @@ -1518,6 +1523,7 @@ void addACLLogEntry(client *c, int reason, int keypos) { * ACL GETUSER * ACL GENPASS * ACL WHOAMI + * ACL LOG [ | RESET] */ void aclCommand(client *c) { char *sub = c->argv[1]->ptr; @@ -1704,6 +1710,36 @@ void aclCommand(client *c) { char pass[32]; /* 128 bits of actual pseudo random data. */ getRandomHexChars(pass,sizeof(pass)); addReplyBulkCBuffer(c,pass,sizeof(pass)); + } else if (!strcasecmp(sub,"log") && (c->argc == 2 || c->argc ==3)) { + long count = 10; /* Number of entries to emit by default. */ + + /* Parse the only argument that LOG may have: it could be either + * the number of entires the user wants to display, or alternatively + * the "RESET" command in order to flush the old entires. */ + if (c->argc == 3) { + if (!strcasecmp(c->argv[2]->ptr,"reset")) { + /* TODO: reset the log. */ + addReply(c,shared.ok); + return; + } else if (getLongFromObjectOrReply(c,c->argv[2],&count,NULL) + != C_OK) + { + return; + } + if (count < 0) count = 0; + } + + /* Fix the count according to the number of entries we got. */ + if ((size_t)count > listLength(ACLLog)) + count = listLength(ACLLog); + + addReplyArrayLen(c,count); + listIter li; + listNode *ln; + listRewind(ACLLog,&li); + while (count-- && (ln = listNext(&li)) != NULL) { + addReplyLongLong(c,1234); + } } else if (!strcasecmp(sub,"help")) { const char *help[] = { "LOAD -- Reload users from the ACL file.", @@ -1716,6 +1752,7 @@ void aclCommand(client *c) { "CAT -- List commands inside category.", "GENPASS -- Generate a secure user password.", "WHOAMI -- Return the current connection username.", +"LOG [ | RESET] -- Show the ACL log entries.", NULL }; addReplyHelp(c,help); From f550dfca478a4351b37ba0894e0fae21420f76c5 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 28 Jan 2020 18:04:20 +0100 Subject: [PATCH 0054/1280] ACL LOG: actually emit entries. --- src/acl.c | 34 ++++++++++++++++++++++++++++++---- src/server.c | 4 +++- src/server.h | 1 + 3 files changed, 34 insertions(+), 5 deletions(-) diff --git a/src/acl.c b/src/acl.c index a166469d0..5937c069c 100644 --- a/src/acl.c +++ b/src/acl.c @@ -1467,7 +1467,7 @@ void ACLLoadUsersAtStartup(void) { #define ACL_LOG_CTX_MULTI 2 /* This structure defines an entry inside the ACL log. */ -typedef struct aclLogEntry { +typedef struct ACLLogEntry { uint64_t count; /* Number of times this happened recently. */ int reason; /* Reason for denying the command. ACL_DENIED_*. */ int context; /* Toplevel, Lua or MULTI/EXEC? ACL_LOG_CTX_*. */ @@ -1475,7 +1475,7 @@ typedef struct aclLogEntry { sds username; /* User the client is authenticated with. */ mstime_t ctime; /* Milliseconds time of last update to this entry. */ sds cinfo; /* Client info (last client if updated). */ -} aclLogEntry; +} ACLLogEntry; /* Adds a new entry in the ACL log, making sure to delete the old entry * if we reach the maximum length allowed for the log. This function attempts @@ -1484,8 +1484,9 @@ typedef struct aclLogEntry { * rules issues. */ void addACLLogEntry(client *c, int reason, int keypos) { /* Create a new entry. */ - struct aclLogEntry *le = zmalloc(sizeof(*le)); + struct ACLLogEntry *le = zmalloc(sizeof(*le)); le->count = 1; + le->reason = reason; le->object = (reason == ACL_DENIED_CMD) ? sdsnew(c->cmd->name) : sdsdup(c->argv[keypos]->ptr); le->username = sdsdup(c->user->name); @@ -1737,8 +1738,33 @@ void aclCommand(client *c) { listIter li; listNode *ln; listRewind(ACLLog,&li); + mstime_t now = mstime(); while (count-- && (ln = listNext(&li)) != NULL) { - addReplyLongLong(c,1234); + ACLLogEntry *le = listNodeValue(ln); + addReplyMapLen(c,7); + addReplyBulkCString(c,"count"); + addReplyLongLong(c,le->count); + addReplyBulkCString(c,"reason"); + addReplyBulkCString(c,(le->reason == ACL_DENIED_CMD) ? + "command" : "key"); + char *ctxstr; + switch(le->context) { + case ACL_LOG_CTX_TOPLEVEL: ctxstr="toplevel"; break; + case ACL_LOG_CTX_MULTI: ctxstr="multi"; break; + case ACL_LOG_CTX_LUA: ctxstr="lua"; break; + default: ctxstr="unknown"; + } + addReplyBulkCString(c,"context"); + addReplyBulkCString(c,ctxstr); + addReplyBulkCString(c,"object"); + addReplyBulkCBuffer(c,le->object,sdslen(le->object)); + addReplyBulkCString(c,"username"); + addReplyBulkCBuffer(c,le->username,sdslen(le->username)); + addReplyBulkCString(c,"age-seconds"); + double age = (double)(now - le->ctime)/1000; + addReplyDouble(c,age); + addReplyBulkCString(c,"client-info"); + addReplyBulkCBuffer(c,le->cinfo,sdslen(le->cinfo)); } } else if (!strcasecmp(sub,"help")) { const char *help[] = { diff --git a/src/server.c b/src/server.c index b5e27e238..6968f311f 100644 --- a/src/server.c +++ b/src/server.c @@ -3377,8 +3377,10 @@ int processCommand(client *c) { /* Check if the user can run this command according to the current * ACLs. */ - int acl_retval = ACLCheckCommandPerm(c,NULL); + int acl_keypos; + int acl_retval = ACLCheckCommandPerm(c,&acl_keypos); if (acl_retval != ACL_OK) { + addACLLogEntry(c,acl_retval,acl_keypos); flagTransaction(c); if (acl_retval == ACL_DENIED_CMD) addReplyErrorFormat(c, diff --git a/src/server.h b/src/server.h index 6fd0ffc5b..bd4aed192 100644 --- a/src/server.h +++ b/src/server.h @@ -1836,6 +1836,7 @@ void ACLLoadUsersAtStartup(void); void addReplyCommandCategories(client *c, struct redisCommand *cmd); user *ACLCreateUnlinkedUser(); void ACLFreeUserAndKillClients(user *u); +void addACLLogEntry(client *c, int reason, int keypos); /* Sorted sets data type */ From 2ed300b78658f79a97ca2af4ef25643617ff9c49 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 29 Jan 2020 12:47:50 +0100 Subject: [PATCH 0055/1280] Add more info in the unblockClientFromModule() function. --- src/module.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/module.c b/src/module.c index d2b267be2..7fdab1b34 100644 --- a/src/module.c +++ b/src/module.c @@ -4282,7 +4282,13 @@ void unblockClientFromModule(client *c) { * not implemented (or it was, but RM_UnblockClient was not called from * within it, as it should). * We must call moduleUnblockClient in order to free privdata and - * RedisModuleBlockedClient */ + * RedisModuleBlockedClient. + * + * Note that clients implementing threads and working with private data, + * should make sure to stop the threads or protect the private data + * in some other way in the disconnection and timeout callback, because + * here we are going to free the private data associated with the + * blocked client. */ if (!bc->unblocked) moduleUnblockClient(c); From 5bcb4e69056618716508da2042f53618b564f881 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 29 Jan 2020 18:40:32 +0100 Subject: [PATCH 0056/1280] ACL LOG: group similar entries in a given time delta. --- src/acl.c | 61 ++++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 58 insertions(+), 3 deletions(-) diff --git a/src/acl.c b/src/acl.c index 5937c069c..d03599a79 100644 --- a/src/acl.c +++ b/src/acl.c @@ -1465,6 +1465,7 @@ void ACLLoadUsersAtStartup(void) { #define ACL_LOG_CTX_TOPLEVEL 0 #define ACL_LOG_CTX_LUA 1 #define ACL_LOG_CTX_MULTI 2 +#define ACL_LOG_GROUPING_MAX_TIME_DELTA 60000 /* This structure defines an entry inside the ACL log. */ typedef struct ACLLogEntry { @@ -1477,6 +1478,28 @@ typedef struct ACLLogEntry { sds cinfo; /* Client info (last client if updated). */ } ACLLogEntry; +/* This function will check if ACL entries 'a' and 'b' are similar enough + * that we should actually update the existing entry in our ACL log instead + * of creating a new one. */ +int ACLLogMatchEntry(ACLLogEntry *a, ACLLogEntry *b) { + if (a->reason != b->reason) return 0; + if (a->context != b->context) return 0; + mstime_t delta = a->ctime - b->ctime; + if (delta < 0) delta = -delta; + if (delta > ACL_LOG_GROUPING_MAX_TIME_DELTA) return 0; + if (sdscmp(a->object,b->object) != 0) return 0; + if (sdscmp(a->username,b->username) != 0) return 0; + return 1; +} + +/* Release an ACL log entry. */ +void ACLFreeLogEntry(ACLLogEntry *le) { + sdsfree(le->object); + sdsfree(le->username); + sdsfree(le->cinfo); + zfree(le); +} + /* Adds a new entry in the ACL log, making sure to delete the old entry * if we reach the maximum length allowed for the log. This function attempts * to find similar entries in the current log in order to bump the counter of @@ -1504,9 +1527,41 @@ void addACLLogEntry(client *c, int reason, int keypos) { le->context = ACL_LOG_CTX_TOPLEVEL; } - /* Add it to our list of entires. We'll have to trim the list - * to its maximum size. */ - listAddNodeHead(ACLLog, le); + /* Try to match this entry with past ones, to see if we can just + * update an existing entry instead of creating a new one. */ + long toscan = 10; /* Do a limited work trying to find duplicated. */ + listIter li; + listNode *ln; + listRewind(ACLLog,&li); + ACLLogEntry *match = NULL; + while (toscan-- && (ln = listNext(&li)) != NULL) { + ACLLogEntry *current = listNodeValue(ln); + if (ACLLogMatchEntry(current,le)) { + match = current; + listDelNode(ACLLog,ln); + listAddNodeHead(ACLLog,current); + break; + } + } + + /* If there is a match update the entry, otherwise add it as a + * new one. */ + if (match) { + /* We update a few fields of the existing entry and bump the + * counter of events for this entry. */ + sdsfree(match->cinfo); + match->cinfo = le->cinfo; + match->ctime = le->ctime; + match->count++; + + /* Release the old entry. */ + le->cinfo = NULL; + ACLFreeLogEntry(le); + } else { + /* Add it to our list of entires. We'll have to trim the list + * to its maximum size. */ + listAddNodeHead(ACLLog, le); + } } /* ============================================================================= From eef1489240d4883cad54e9c3a5f0564fe700f727 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 29 Jan 2020 18:51:04 +0100 Subject: [PATCH 0057/1280] ACL LOG: implement LOG RESET. --- src/acl.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/acl.c b/src/acl.c index d03599a79..16acd4d3e 100644 --- a/src/acl.c +++ b/src/acl.c @@ -95,6 +95,7 @@ struct ACLUserFlag { void ACLResetSubcommandsForCommand(user *u, unsigned long id); void ACLResetSubcommands(user *u); void ACLAddAllowedSubcommand(user *u, unsigned long id, const char *sub); +void ACLFreeLogEntry(void *le); /* The length of the string representation of a hashed password. */ #define HASH_PASSWORD_LEN SHA256_BLOCK_SIZE*2 @@ -1493,7 +1494,8 @@ int ACLLogMatchEntry(ACLLogEntry *a, ACLLogEntry *b) { } /* Release an ACL log entry. */ -void ACLFreeLogEntry(ACLLogEntry *le) { +void ACLFreeLogEntry(void *leptr) { + ACLLogEntry *le = leptr; sdsfree(le->object); sdsfree(le->username); sdsfree(le->cinfo); @@ -1774,7 +1776,9 @@ void aclCommand(client *c) { * the "RESET" command in order to flush the old entires. */ if (c->argc == 3) { if (!strcasecmp(c->argv[2]->ptr,"reset")) { - /* TODO: reset the log. */ + listSetFreeMethod(ACLLog,ACLFreeLogEntry); + listEmpty(ACLLog); + listSetFreeMethod(ACLLog,NULL); addReply(c,shared.ok); return; } else if (getLongFromObjectOrReply(c,c->argv[2],&count,NULL) From a7420b2478bb0c38889a034617efffa92bb1f14e Mon Sep 17 00:00:00 2001 From: Yossi Gottlieb Date: Wed, 29 Jan 2020 21:40:02 +0200 Subject: [PATCH 0058/1280] TLS: Fix missing initialization in redis-cli. --- src/redis-cli.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/redis-cli.c b/src/redis-cli.c index 065c389c6..1919829e1 100644 --- a/src/redis-cli.c +++ b/src/redis-cli.c @@ -49,6 +49,7 @@ #include #ifdef USE_OPENSSL #include +#include #include #endif #include /* use sds.h from hiredis, so that only one set of sds functions will be present in the binary */ @@ -7933,6 +7934,14 @@ int main(int argc, char **argv) { parseEnv(); +#ifdef USE_OPENSSL + if (config.tls) { + ERR_load_crypto_strings(); + SSL_load_error_strings(); + SSL_library_init(); + } +#endif + /* Cluster Manager mode */ if (CLUSTER_MANAGER_MODE()) { clusterManagerCommandProc *proc = validateClusterManagerCommand(); From bc168c91ecdbfd32b1d77687460f6b730f9584d0 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 30 Jan 2020 10:50:32 +0100 Subject: [PATCH 0059/1280] ACL LOG: also log ACL errors in the scripting/MULTI ctx. --- src/multi.c | 4 +++- src/scripting.c | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/multi.c b/src/multi.c index 640149870..a88e5336b 100644 --- a/src/multi.c +++ b/src/multi.c @@ -177,8 +177,10 @@ void execCommand(client *c) { must_propagate = 1; } - int acl_retval = ACLCheckCommandPerm(c,NULL); + int acl_keypos; + int acl_retval = ACLCheckCommandPerm(c,&acl_keypos); if (acl_retval != ACL_OK) { + addACLLogEntry(c,acl_retval,acl_keypos); addReplyErrorFormat(c, "-NOPERM ACLs rules changed between the moment the " "transaction was accumulated and the EXEC call. " diff --git a/src/scripting.c b/src/scripting.c index 0e47ebcb0..916b90cf1 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -606,8 +606,10 @@ int luaRedisGenericCommand(lua_State *lua, int raise_error) { } /* Check the ACLs. */ - int acl_retval = ACLCheckCommandPerm(c,NULL); + int acl_keypos; + int acl_retval = ACLCheckCommandPerm(c,&acl_keypos); if (acl_retval != ACL_OK) { + addACLLogEntry(c,acl_retval,acl_keypos); if (acl_retval == ACL_DENIED_CMD) luaPushError(lua, "The user executing the script can't run this " "command or subcommand"); From 8185f2b6ceb26138778d4cf3584012a261d42a78 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 30 Jan 2020 11:09:50 +0100 Subject: [PATCH 0060/1280] ACL LOG: implement a few basic tests. --- tests/unit/acl.tcl | 87 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) diff --git a/tests/unit/acl.tcl b/tests/unit/acl.tcl index 2205d2d86..bd909d36c 100644 --- a/tests/unit/acl.tcl +++ b/tests/unit/acl.tcl @@ -141,4 +141,91 @@ start_server {tags {"acl"}} { r ACL setuser newuser -debug # The test framework will detect a leak if any. } + + test {ACL LOG shows failed command executions at toplevel} { + r ACL LOG RESET + r ACL setuser antirez >foo on +set ~object:1234 + r ACL setuser antirez +eval +multi +exec + r AUTH antirez foo + catch {r GET foo} + r AUTH default "" + set entry [lindex [r ACL LOG] 0] + assert {[dict get $entry username] eq {antirez}} + assert {[dict get $entry context] eq {toplevel}} + assert {[dict get $entry reason] eq {command}} + assert {[dict get $entry object] eq {get}} + } + + test {ACL LOG is able to test similar events} { + r AUTH antirez foo + catch {r GET foo} + catch {r GET foo} + catch {r GET foo} + r AUTH default "" + set entry [lindex [r ACL LOG] 0] + assert {[dict get $entry count] == 4} + } + + test {ACL LOG is able to log keys access violations and key name} { + r AUTH antirez foo + catch {r SET somekeynotallowed 1234} + r AUTH default "" + set entry [lindex [r ACL LOG] 0] + assert {[dict get $entry reason] eq {key}} + assert {[dict get $entry object] eq {somekeynotallowed}} + } + + test {ACL LOG RESET is able to flush the entries in the log} { + r ACL LOG RESET + assert {[llength [r ACL LOG]] == 0} + } + + test {ACL LOG can distinguish the transaction context (1)} { + r AUTH antirez foo + r MULTI + catch {r INCR foo} + catch {r EXEC} + r AUTH default "" + set entry [lindex [r ACL LOG] 0] + assert {[dict get $entry context] eq {multi}} + assert {[dict get $entry object] eq {incr}} + } + + test {ACL LOG can distinguish the transaction context (2)} { + set rd1 [redis_deferring_client] + r ACL SETUSER antirez +incr + + r AUTH antirez foo + r MULTI + r INCR object:1234 + $rd1 ACL SETUSER antirez -incr + $rd1 read + catch {r EXEC} + $rd1 close + r AUTH default "" + set entry [lindex [r ACL LOG] 0] + assert {[dict get $entry context] eq {multi}} + assert {[dict get $entry object] eq {incr}} + r ACL SETUSER antirez -incr + } + + test {ACL can log errors in the context of Lua scripting} { + r AUTH antirez foo + catch {r EVAL {redis.call('incr','foo')} 0} + r AUTH default "" + set entry [lindex [r ACL LOG] 0] + assert {[dict get $entry context] eq {lua}} + assert {[dict get $entry object] eq {incr}} + } + + test {ACL LOG can accept a numerical argument to show less entries} { + r AUTH antirez foo + catch {r INCR foo} + catch {r INCR foo} + catch {r INCR foo} + catch {r INCR foo} + r AUTH default "" + assert {[llength [r ACL LOG]] > 1} + assert {[llength [r ACL LOG 2]] == 2} + } } From 3fd717daeead10f79971049658cfd18b5ffdf301 Mon Sep 17 00:00:00 2001 From: Guy Benoish Date: Thu, 30 Jan 2020 18:14:45 +0530 Subject: [PATCH 0061/1280] ld2string should fail if string contains \0 in the middle This bug affected RM_StringToLongDouble and HINCRBYFLOAT. I added tests for both cases. Main changes: 1. Fixed string2ld to fail if string contains \0 in the middle 2. Use string2ld in getLongDoubleFromObject - No point of having duplicated code here The two changes above broke RM_SaveLongDouble/RM_LoadLongDouble because the long double string was saved with length+1 (An innocent mistake, but it's actually a bug - The length passed to RM_SaveLongDouble should not include the last \0). --- src/module.c | 2 +- src/object.c | 10 +--------- src/util.c | 3 ++- tests/modules/misc.c | 9 +++++++++ tests/unit/type/hash.tcl | 7 +++++++ 5 files changed, 20 insertions(+), 11 deletions(-) diff --git a/src/module.c b/src/module.c index d2b267be2..584912fbe 100644 --- a/src/module.c +++ b/src/module.c @@ -3901,7 +3901,7 @@ void RM_SaveLongDouble(RedisModuleIO *io, long double value) { /* Long double has different number of bits in different platforms, so we * save it as a string type. */ size_t len = ld2string(buf,sizeof(buf),value,LD_STR_HEX); - RM_SaveStringBuffer(io,buf,len+1); /* len+1 for '\0' */ + RM_SaveStringBuffer(io,buf,len); } /* In the context of the rdb_save method of a module data type, loads back the diff --git a/src/object.c b/src/object.c index 2201a317a..52007b474 100644 --- a/src/object.c +++ b/src/object.c @@ -640,21 +640,13 @@ int getDoubleFromObjectOrReply(client *c, robj *o, double *target, const char *m int getLongDoubleFromObject(robj *o, long double *target) { long double value; - char *eptr; if (o == NULL) { value = 0; } else { serverAssertWithInfo(NULL,o,o->type == OBJ_STRING); if (sdsEncodedObject(o)) { - errno = 0; - value = strtold(o->ptr, &eptr); - if (sdslen(o->ptr) == 0 || - isspace(((const char*)o->ptr)[0]) || - (size_t)(eptr-(char*)o->ptr) != sdslen(o->ptr) || - (errno == ERANGE && - (value == HUGE_VAL || value == -HUGE_VAL || value == 0)) || - isnan(value)) + if (!string2ld(o->ptr, sdslen(o->ptr), &value)) return C_ERR; } else if (o->encoding == OBJ_ENCODING_INT) { value = (long)o->ptr; diff --git a/src/util.c b/src/util.c index 20471b539..2be42a0df 100644 --- a/src/util.c +++ b/src/util.c @@ -471,13 +471,14 @@ int string2ld(const char *s, size_t slen, long double *dp) { long double value; char *eptr; - if (slen >= sizeof(buf)) return 0; + if (slen == 0 || slen >= sizeof(buf)) return 0; memcpy(buf,s,slen); buf[slen] = '\0'; errno = 0; value = strtold(buf, &eptr); if (isspace(buf[0]) || eptr[0] != '\0' || + (size_t)(eptr-buf) != slen || (errno == ERANGE && (value == HUGE_VAL || value == -HUGE_VAL || value == 0)) || errno == EINVAL || diff --git a/tests/modules/misc.c b/tests/modules/misc.c index b5a032f60..41bec06ed 100644 --- a/tests/modules/misc.c +++ b/tests/modules/misc.c @@ -74,6 +74,15 @@ int test_ld_conv(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { RedisModule_ReplyWithError(ctx, err); goto final; } + /* Make sure we can't convert a string that has \0 in it */ + char buf[4] = "123"; + buf[1] = '\0'; + RedisModuleString *s3 = RedisModule_CreateString(ctx, buf, 3); + long double ld3; + if (RedisModule_StringToLongDouble(s3, &ld3) == REDISMODULE_OK) { + RedisModule_ReplyWithError(ctx, "Invalid string successfully converted to long double"); + goto final; + } RedisModule_ReplyWithLongDouble(ctx, ld2); final: RedisModule_FreeString(ctx, s1); diff --git a/tests/unit/type/hash.tcl b/tests/unit/type/hash.tcl index d2c679d32..9f8a21b1c 100644 --- a/tests/unit/type/hash.tcl +++ b/tests/unit/type/hash.tcl @@ -390,6 +390,13 @@ start_server {tags {"hash"}} { lappend rv [string match "ERR*not*float*" $bigerr] } {1 1} + test {HINCRBYFLOAT fails against hash value that contains a null-terminator in the middle} { + r hset h f "1\x002" + catch {r hincrbyfloat h f 1} err + set rv {} + lappend rv [string match "ERR*not*float*" $err] + } {1} + test {HSTRLEN against the small hash} { set err {} foreach k [array names smallhash *] { From add0229c317db9778acf47231283dc11fbd16bec Mon Sep 17 00:00:00 2001 From: Guy Benoish Date: Thu, 30 Jan 2020 19:15:12 +0530 Subject: [PATCH 0062/1280] DEBUG OBJECT should pass keyname to module when loading --- src/debug.c | 2 +- src/rdb.c | 4 ++-- src/rdb.h | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/debug.c b/src/debug.c index a2d37337d..fecee25e1 100644 --- a/src/debug.c +++ b/src/debug.c @@ -488,7 +488,7 @@ NULL "encoding:%s serializedlength:%zu " "lru:%d lru_seconds_idle:%llu%s", (void*)val, val->refcount, - strenc, rdbSavedObjectLen(val), + strenc, rdbSavedObjectLen(val, c->argv[2]), val->lru, estimateObjectIdleTime(val)/1000, extra); } else if (!strcasecmp(c->argv[1]->ptr,"sdslen") && c->argc == 3) { dictEntry *de; diff --git a/src/rdb.c b/src/rdb.c index 27e1b3135..de3cdb459 100644 --- a/src/rdb.c +++ b/src/rdb.c @@ -1002,8 +1002,8 @@ ssize_t rdbSaveObject(rio *rdb, robj *o, robj *key) { * the rdbSaveObject() function. Currently we use a trick to get * this length with very little changes to the code. In the future * we could switch to a faster solution. */ -size_t rdbSavedObjectLen(robj *o) { - ssize_t len = rdbSaveObject(NULL,o,NULL); +size_t rdbSavedObjectLen(robj *o, robj *key) { + ssize_t len = rdbSaveObject(NULL,o,key); serverAssertWithInfo(NULL,o,len != -1); return len; } diff --git a/src/rdb.h b/src/rdb.h index 4229beea8..b276a978b 100644 --- a/src/rdb.h +++ b/src/rdb.h @@ -143,7 +143,7 @@ int rdbSaveToSlavesSockets(rdbSaveInfo *rsi); void rdbRemoveTempFile(pid_t childpid); int rdbSave(char *filename, rdbSaveInfo *rsi); ssize_t rdbSaveObject(rio *rdb, robj *o, robj *key); -size_t rdbSavedObjectLen(robj *o); +size_t rdbSavedObjectLen(robj *o, robj *key); robj *rdbLoadObject(int type, rio *rdb, robj *key); void backgroundSaveDoneHandler(int exitcode, int bysignal); int rdbSaveKeyValuePair(rio *rdb, robj *key, robj *val, long long expiretime); From 0ab1604c0c8cdc464df43fe0ec344b08062a906a Mon Sep 17 00:00:00 2001 From: Leo Murillo Date: Sun, 2 Feb 2020 02:48:00 -0600 Subject: [PATCH 0063/1280] Set ZSKIPLIST_MAXLEVEL to optimal value given 2^64 elements and p=0.25 --- src/server.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/server.h b/src/server.h index 8e354c03d..5d63e9b55 100644 --- a/src/server.h +++ b/src/server.h @@ -335,7 +335,7 @@ typedef long long ustime_t; /* microsecond time type. */ /* Anti-warning macro... */ #define UNUSED(V) ((void) V) -#define ZSKIPLIST_MAXLEVEL 64 /* Should be enough for 2^64 elements */ +#define ZSKIPLIST_MAXLEVEL 32 /* Should be enough for 2^64 elements */ #define ZSKIPLIST_P 0.25 /* Skiplist P = 1/4 */ /* Append only defines */ From 2492bdc90b91be8fe240fad3796de137c321fc82 Mon Sep 17 00:00:00 2001 From: Guy Benoish Date: Mon, 3 Feb 2020 15:19:44 +0530 Subject: [PATCH 0064/1280] Exclude "keymiss" notification from NOTIFY_ALL Because "keymiss" is "special" compared to the rest of the notifications (Trying not to break existing apps using the 'A' format for notifications) Also updated redis.conf and module.c docs --- redis.conf | 6 +++++- src/module.c | 3 ++- src/notify.c | 2 +- src/redismodule.h | 4 ++-- src/server.h | 4 ++-- 5 files changed, 12 insertions(+), 7 deletions(-) diff --git a/redis.conf b/redis.conf index 07005cffe..2b8711eea 100644 --- a/redis.conf +++ b/redis.conf @@ -1362,7 +1362,11 @@ latency-monitor-threshold 0 # z Sorted set commands # x Expired events (events generated every time a key expires) # e Evicted events (events generated when a key is evicted for maxmemory) -# A Alias for g$lshzxe, so that the "AKE" string means all the events. +# t Stream commands +# m Key-miss events (Note: It is not included in the 'A' class) +# A Alias for g$lshzxet, so that the "AKE" string means all the events +# (Except key-miss events which are excluded from 'A' due to their +# unique nature). # # The "notify-keyspace-events" takes as argument a string that is composed # of zero or multiple characters. The empty string means that notifications diff --git a/src/module.c b/src/module.c index d2b267be2..54072af57 100644 --- a/src/module.c +++ b/src/module.c @@ -4798,7 +4798,8 @@ void moduleReleaseGIL(void) { * - REDISMODULE_NOTIFY_EXPIRED: Expiration events * - REDISMODULE_NOTIFY_EVICTED: Eviction events * - REDISMODULE_NOTIFY_STREAM: Stream events - * - REDISMODULE_NOTIFY_ALL: All events + * - REDISMODULE_NOTIFY_KEYMISS: Key-miss events + * - REDISMODULE_NOTIFY_ALL: All events (Excluding REDISMODULE_NOTIFY_KEYMISS) * * We do not distinguish between key events and keyspace events, and it is up * to the module to filter the actions taken based on the key. diff --git a/src/notify.c b/src/notify.c index d6c3ad403..bb1055724 100644 --- a/src/notify.c +++ b/src/notify.c @@ -82,10 +82,10 @@ sds keyspaceEventsFlagsToString(int flags) { if (flags & NOTIFY_EXPIRED) res = sdscatlen(res,"x",1); if (flags & NOTIFY_EVICTED) res = sdscatlen(res,"e",1); if (flags & NOTIFY_STREAM) res = sdscatlen(res,"t",1); - if (flags & NOTIFY_KEY_MISS) res = sdscatlen(res,"m",1); } if (flags & NOTIFY_KEYSPACE) res = sdscatlen(res,"K",1); if (flags & NOTIFY_KEYEVENT) res = sdscatlen(res,"E",1); + if (flags & NOTIFY_KEY_MISS) res = sdscatlen(res,"m",1); return res; } diff --git a/src/redismodule.h b/src/redismodule.h index 637078f2b..e74611f13 100644 --- a/src/redismodule.h +++ b/src/redismodule.h @@ -127,8 +127,8 @@ #define REDISMODULE_NOTIFY_EXPIRED (1<<8) /* x */ #define REDISMODULE_NOTIFY_EVICTED (1<<9) /* e */ #define REDISMODULE_NOTIFY_STREAM (1<<10) /* t */ -#define REDISMODULE_NOTIFY_KEY_MISS (1<<11) /* m */ -#define REDISMODULE_NOTIFY_ALL (REDISMODULE_NOTIFY_GENERIC | REDISMODULE_NOTIFY_STRING | REDISMODULE_NOTIFY_LIST | REDISMODULE_NOTIFY_SET | REDISMODULE_NOTIFY_HASH | REDISMODULE_NOTIFY_ZSET | REDISMODULE_NOTIFY_EXPIRED | REDISMODULE_NOTIFY_EVICTED | REDISMODULE_NOTIFY_STREAM | REDISMODULE_NOTIFY_KEY_MISS) /* A */ +#define REDISMODULE_NOTIFY_KEY_MISS (1<<11) /* m (Note: This one is excluded from REDISMODULE_NOTIFY_ALL on purpose) */ +#define REDISMODULE_NOTIFY_ALL (REDISMODULE_NOTIFY_GENERIC | REDISMODULE_NOTIFY_STRING | REDISMODULE_NOTIFY_LIST | REDISMODULE_NOTIFY_SET | REDISMODULE_NOTIFY_HASH | REDISMODULE_NOTIFY_ZSET | REDISMODULE_NOTIFY_EXPIRED | REDISMODULE_NOTIFY_EVICTED | REDISMODULE_NOTIFY_STREAM) /* A */ /* A special pointer that we can use between the core and the module to signal diff --git a/src/server.h b/src/server.h index 8e354c03d..257f22985 100644 --- a/src/server.h +++ b/src/server.h @@ -413,8 +413,8 @@ typedef long long ustime_t; /* microsecond time type. */ #define NOTIFY_EXPIRED (1<<8) /* x */ #define NOTIFY_EVICTED (1<<9) /* e */ #define NOTIFY_STREAM (1<<10) /* t */ -#define NOTIFY_KEY_MISS (1<<11) /* m */ -#define NOTIFY_ALL (NOTIFY_GENERIC | NOTIFY_STRING | NOTIFY_LIST | NOTIFY_SET | NOTIFY_HASH | NOTIFY_ZSET | NOTIFY_EXPIRED | NOTIFY_EVICTED | NOTIFY_STREAM | NOTIFY_KEY_MISS) /* A flag */ +#define NOTIFY_KEY_MISS (1<<11) /* m (Note: This one is excluded from NOTIFY_ALL on purpose) */ +#define NOTIFY_ALL (NOTIFY_GENERIC | NOTIFY_STRING | NOTIFY_LIST | NOTIFY_SET | NOTIFY_HASH | NOTIFY_ZSET | NOTIFY_EXPIRED | NOTIFY_EVICTED | NOTIFY_STREAM) /* A flag */ /* Get the first bind addr or NULL */ #define NET_FIRST_BIND_ADDR (server.bindaddr_count ? server.bindaddr[0] : NULL) From f5d37082ca2e8c58256f031397dd75592add9478 Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Mon, 3 Feb 2020 15:58:28 +0200 Subject: [PATCH 0065/1280] fix uninitialized info_cb var in module.c --- src/module.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/module.c b/src/module.c index d922c5c20..914c50df3 100644 --- a/src/module.c +++ b/src/module.c @@ -859,6 +859,7 @@ void RM_SetModuleAttribs(RedisModuleCtx *ctx, const char *name, int ver, int api module->in_call = 0; module->in_hook = 0; module->options = 0; + module->info_cb = 0; ctx->module = module; } From c0c29adce94435474f8ef04266ca77533e78e971 Mon Sep 17 00:00:00 2001 From: WuYunlong Date: Tue, 4 Feb 2020 16:34:11 +0800 Subject: [PATCH 0066/1280] Add tcl regression test in scripting.tcl to reproduce memory leak. --- tests/unit/scripting.tcl | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/unit/scripting.tcl b/tests/unit/scripting.tcl index 2543a0377..fb36d0b80 100644 --- a/tests/unit/scripting.tcl +++ b/tests/unit/scripting.tcl @@ -741,3 +741,8 @@ start_server {tags {"scripting repl"}} { } } +start_server {tags {"scripting"}} { + r script debug sync + r eval {return 'hello'} 0 + r eval {return 'hello'} 0 +} From 85bc26f8da7a8dab5d8aa8da3c5b01f50793475c Mon Sep 17 00:00:00 2001 From: WuYunlong Date: Tue, 4 Feb 2020 16:38:46 +0800 Subject: [PATCH 0067/1280] Fix lua related memory leak. --- src/scripting.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/scripting.c b/src/scripting.c index 9282b7fd9..a5c59b113 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -2473,6 +2473,7 @@ void ldbEval(lua_State *lua, sds *argv, int argc) { ldbLog(sdscatfmt(sdsempty()," %s",lua_tostring(lua,-1))); lua_pop(lua,1); sdsfree(code); + sdsfree(expr); return; } } From 54199c93bed390e181688d2761879d0de794947a Mon Sep 17 00:00:00 2001 From: lifubang Date: Tue, 4 Feb 2020 17:32:30 +0800 Subject: [PATCH 0068/1280] fix ssl flag check for redis-cli Signed-off-by: lifubang --- src/redis-cli.c | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/src/redis-cli.c b/src/redis-cli.c index 1919829e1..c1bda0a00 100644 --- a/src/redis-cli.c +++ b/src/redis-cli.c @@ -1595,15 +1595,15 @@ static int parseOptions(int argc, char **argv) { #ifdef USE_OPENSSL } else if (!strcmp(argv[i],"--tls")) { config.tls = 1; - } else if (!strcmp(argv[i],"--sni")) { + } else if (!strcmp(argv[i],"--sni") && !lastarg) { config.sni = argv[++i]; - } else if (!strcmp(argv[i],"--cacertdir")) { + } else if (!strcmp(argv[i],"--cacertdir") && !lastarg) { config.cacertdir = argv[++i]; - } else if (!strcmp(argv[i],"--cacert")) { + } else if (!strcmp(argv[i],"--cacert") && !lastarg) { config.cacert = argv[++i]; - } else if (!strcmp(argv[i],"--cert")) { + } else if (!strcmp(argv[i],"--cert") && !lastarg) { config.cert = argv[++i]; - } else if (!strcmp(argv[i],"--key")) { + } else if (!strcmp(argv[i],"--key") && !lastarg) { config.key = argv[++i]; #endif } else if (!strcmp(argv[i],"-v") || !strcmp(argv[i], "--version")) { @@ -1701,12 +1701,13 @@ static void usage(void) { " -c Enable cluster mode (follow -ASK and -MOVED redirections).\n" #ifdef USE_OPENSSL " --tls Establish a secure TLS connection.\n" -" --cacert CA Certificate file to verify with.\n" -" --cacertdir Directory where trusted CA certificates are stored.\n" +" --sni Server name indication for TLS.\n" +" --cacert CA Certificate file to verify with.\n" +" --cacertdir Directory where trusted CA certificates are stored.\n" " If neither cacert nor cacertdir are specified, the default\n" " system-wide trusted root certs configuration will apply.\n" -" --cert Client certificate to authenticate with.\n" -" --key Private key file to authenticate with.\n" +" --cert Client certificate to authenticate with.\n" +" --key Private key file to authenticate with.\n" #endif " --raw Use raw formatting for replies (default when STDOUT is\n" " not a tty).\n" From debc5d4ef9f56acbf9cb5a148468e36ca92ccdae Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 4 Feb 2020 12:55:26 +0100 Subject: [PATCH 0069/1280] ACL LOG: log failed auth attempts. --- src/acl.c | 37 +++++++++++++++++++++++++++++-------- src/multi.c | 2 +- src/scripting.c | 2 +- src/server.c | 2 +- src/server.h | 3 ++- 5 files changed, 34 insertions(+), 12 deletions(-) diff --git a/src/acl.c b/src/acl.c index 16acd4d3e..97c00d4f8 100644 --- a/src/acl.c +++ b/src/acl.c @@ -982,6 +982,7 @@ int ACLAuthenticateUser(client *c, robj *username, robj *password) { moduleNotifyUserChanged(c); return C_OK; } else { + addACLLogEntry(c,ACL_DENIED_AUTH,0,username->ptr); return C_ERR; } } @@ -1506,17 +1507,29 @@ void ACLFreeLogEntry(void *leptr) { * if we reach the maximum length allowed for the log. This function attempts * to find similar entries in the current log in order to bump the counter of * the log entry instead of creating many entries for very similar ACL - * rules issues. */ -void addACLLogEntry(client *c, int reason, int keypos) { + * rules issues. + * + * The keypos argument is only used when the reason is ACL_DENIED_KEY, since + * it allows the function to log the key name that caused the problem. + * Similarly the username is only passed when we failed to authenticate the + * user with AUTH or HELLO, for the ACL_DENIED_AUTH reason. Otherwise + * it will just be NULL. + */ +void addACLLogEntry(client *c, int reason, int keypos, sds username) { /* Create a new entry. */ struct ACLLogEntry *le = zmalloc(sizeof(*le)); le->count = 1; le->reason = reason; - le->object = (reason == ACL_DENIED_CMD) ? sdsnew(c->cmd->name) : - sdsdup(c->argv[keypos]->ptr); - le->username = sdsdup(c->user->name); + le->username = sdsdup(reason == ACL_DENIED_AUTH ? username : c->user->name); le->ctime = mstime(); + switch(reason) { + case ACL_DENIED_CMD: le->object = sdsnew(c->cmd->name); break; + case ACL_DENIED_KEY: le->object = sdsnew(c->argv[keypos]->ptr); break; + case ACL_DENIED_AUTH: le->object = sdsnew(c->argv[0]->ptr); break; + default: le->object = sdsempty(); + } + client *realclient = c; if (realclient->flags & CLIENT_LUA) realclient = server.lua_caller; @@ -1803,9 +1816,17 @@ void aclCommand(client *c) { addReplyMapLen(c,7); addReplyBulkCString(c,"count"); addReplyLongLong(c,le->count); + addReplyBulkCString(c,"reason"); - addReplyBulkCString(c,(le->reason == ACL_DENIED_CMD) ? - "command" : "key"); + char *reasonstr; + switch(le->reason) { + case ACL_DENIED_CMD: reasonstr="command"; break; + case ACL_DENIED_KEY: reasonstr="key"; break; + case ACL_DENIED_AUTH: reasonstr="auth"; break; + } + addReplyBulkCString(c,reasonstr); + + addReplyBulkCString(c,"context"); char *ctxstr; switch(le->context) { case ACL_LOG_CTX_TOPLEVEL: ctxstr="toplevel"; break; @@ -1813,8 +1834,8 @@ void aclCommand(client *c) { case ACL_LOG_CTX_LUA: ctxstr="lua"; break; default: ctxstr="unknown"; } - addReplyBulkCString(c,"context"); addReplyBulkCString(c,ctxstr); + addReplyBulkCString(c,"object"); addReplyBulkCBuffer(c,le->object,sdslen(le->object)); addReplyBulkCString(c,"username"); diff --git a/src/multi.c b/src/multi.c index a88e5336b..cbbd2c513 100644 --- a/src/multi.c +++ b/src/multi.c @@ -180,7 +180,7 @@ void execCommand(client *c) { int acl_keypos; int acl_retval = ACLCheckCommandPerm(c,&acl_keypos); if (acl_retval != ACL_OK) { - addACLLogEntry(c,acl_retval,acl_keypos); + addACLLogEntry(c,acl_retval,acl_keypos,NULL); addReplyErrorFormat(c, "-NOPERM ACLs rules changed between the moment the " "transaction was accumulated and the EXEC call. " diff --git a/src/scripting.c b/src/scripting.c index 916b90cf1..2eac56b47 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -609,7 +609,7 @@ int luaRedisGenericCommand(lua_State *lua, int raise_error) { int acl_keypos; int acl_retval = ACLCheckCommandPerm(c,&acl_keypos); if (acl_retval != ACL_OK) { - addACLLogEntry(c,acl_retval,acl_keypos); + addACLLogEntry(c,acl_retval,acl_keypos,NULL); if (acl_retval == ACL_DENIED_CMD) luaPushError(lua, "The user executing the script can't run this " "command or subcommand"); diff --git a/src/server.c b/src/server.c index 6968f311f..2827188e0 100644 --- a/src/server.c +++ b/src/server.c @@ -3380,7 +3380,7 @@ int processCommand(client *c) { int acl_keypos; int acl_retval = ACLCheckCommandPerm(c,&acl_keypos); if (acl_retval != ACL_OK) { - addACLLogEntry(c,acl_retval,acl_keypos); + addACLLogEntry(c,acl_retval,acl_keypos,NULL); flagTransaction(c); if (acl_retval == ACL_DENIED_CMD) addReplyErrorFormat(c, diff --git a/src/server.h b/src/server.h index bd4aed192..637ceec1e 100644 --- a/src/server.h +++ b/src/server.h @@ -1820,6 +1820,7 @@ void ACLInit(void); #define ACL_OK 0 #define ACL_DENIED_CMD 1 #define ACL_DENIED_KEY 2 +#define ACL_DENIED_AUTH 3 /* Only used for ACL LOG entries. */ int ACLCheckUserCredentials(robj *username, robj *password); int ACLAuthenticateUser(client *c, robj *username, robj *password); unsigned long ACLGetCommandID(const char *cmdname); @@ -1836,7 +1837,7 @@ void ACLLoadUsersAtStartup(void); void addReplyCommandCategories(client *c, struct redisCommand *cmd); user *ACLCreateUnlinkedUser(); void ACLFreeUserAndKillClients(user *u); -void addACLLogEntry(client *c, int reason, int keypos); +void addACLLogEntry(client *c, int reason, int keypos, sds username); /* Sorted sets data type */ From da054f14f0442e1eb0217db4fa0c42019fb6da4f Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 4 Feb 2020 12:58:48 +0100 Subject: [PATCH 0070/1280] ACL LOG: test for AUTH reason. --- tests/unit/acl.tcl | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/unit/acl.tcl b/tests/unit/acl.tcl index bd909d36c..0e6d5c66a 100644 --- a/tests/unit/acl.tcl +++ b/tests/unit/acl.tcl @@ -228,4 +228,13 @@ start_server {tags {"acl"}} { assert {[llength [r ACL LOG]] > 1} assert {[llength [r ACL LOG 2]] == 2} } + + test {ACL LOG can log failed auth attempts} { + catch {r AUTH antirez wrong-password} + set entry [lindex [r ACL LOG] 0] + assert {[dict get $entry context] eq {toplevel}} + assert {[dict get $entry reason] eq {auth}} + assert {[dict get $entry object] eq {AUTH}} + assert {[dict get $entry username] eq {antirez}} + } } From f1518f63b169153c3c174e2808da801057423d67 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 4 Feb 2020 13:19:40 +0100 Subject: [PATCH 0071/1280] ACL LOG: make max log entries configurable. --- src/acl.c | 6 ++++++ src/config.c | 1 + src/server.h | 1 + tests/unit/acl.tcl | 11 +++++++++++ 4 files changed, 19 insertions(+) diff --git a/src/acl.c b/src/acl.c index 97c00d4f8..fa57e210c 100644 --- a/src/acl.c +++ b/src/acl.c @@ -1576,6 +1576,12 @@ void addACLLogEntry(client *c, int reason, int keypos, sds username) { /* Add it to our list of entires. We'll have to trim the list * to its maximum size. */ listAddNodeHead(ACLLog, le); + while(listLength(ACLLog) > server.acllog_max_len) { + listNode *ln = listLast(ACLLog); + ACLLogEntry *le = listNodeValue(ln); + ACLFreeLogEntry(le); + listDelNode(ACLLog,ln); + } } } diff --git a/src/config.c b/src/config.c index 0526de84d..68a9b0c0d 100644 --- a/src/config.c +++ b/src/config.c @@ -2233,6 +2233,7 @@ standardConfig configs[] = { /* Unsigned Long configs */ createULongConfig("active-defrag-max-scan-fields", NULL, MODIFIABLE_CONFIG, 1, LONG_MAX, server.active_defrag_max_scan_fields, 1000, INTEGER_CONFIG, NULL, NULL), /* Default: keys with more than 1000 fields will be processed separately */ createULongConfig("slowlog-max-len", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.slowlog_max_len, 128, INTEGER_CONFIG, NULL, NULL), + createULongConfig("acllog-max-len", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.acllog_max_len, 128, INTEGER_CONFIG, NULL, NULL), /* Long Long configs */ createLongLongConfig("lua-time-limit", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.lua_time_limit, 5000, INTEGER_CONFIG, NULL, NULL),/* milliseconds */ diff --git a/src/server.h b/src/server.h index 637ceec1e..f2040436c 100644 --- a/src/server.h +++ b/src/server.h @@ -1385,6 +1385,7 @@ struct redisServer { dict *latency_events; /* ACLs */ char *acl_filename; /* ACL Users file. NULL if not configured. */ + unsigned long acllog_max_len; /* Maximum length of the ACL LOG list. */ /* Assert & bug reporting */ const char *assert_failed; const char *assert_file; diff --git a/tests/unit/acl.tcl b/tests/unit/acl.tcl index 0e6d5c66a..fc1664a75 100644 --- a/tests/unit/acl.tcl +++ b/tests/unit/acl.tcl @@ -237,4 +237,15 @@ start_server {tags {"acl"}} { assert {[dict get $entry object] eq {AUTH}} assert {[dict get $entry username] eq {antirez}} } + + test {ACL LOG entries are limited to a maximum amount} { + r ACL LOG RESET + r CONFIG SET acllog-max-len 5 + r AUTH antirez foo + for {set j 0} {$j < 10} {incr j} { + catch {r SET obj:$j 123} + } + r AUTH default "" + assert {[llength [r ACL LOG]] == 5} + } } From a0d45826a38a5b385875a8c87e6b3aef14f79591 Mon Sep 17 00:00:00 2001 From: Guy Benoish Date: Tue, 4 Feb 2020 19:28:09 +0530 Subject: [PATCH 0072/1280] Add RM_CreateStringFromDouble --- src/module.c | 12 ++++++++++++ src/redismodule.h | 2 ++ 2 files changed, 14 insertions(+) diff --git a/src/module.c b/src/module.c index d2b267be2..91437c6a6 100644 --- a/src/module.c +++ b/src/module.c @@ -1037,6 +1037,17 @@ RedisModuleString *RM_CreateStringFromLongLong(RedisModuleCtx *ctx, long long ll return RM_CreateString(ctx,buf,len); } +/* Like RedisModule_CreatString(), but creates a string starting from a double + * integer instead of taking a buffer and its length. + * + * The returned string must be released with RedisModule_FreeString() or by + * enabling automatic memory management. */ +RedisModuleString *RM_CreateStringFromDouble(RedisModuleCtx *ctx, double d) { + char buf[128]; + size_t len = d2string(buf,sizeof(buf),d); + return RM_CreateString(ctx,buf,len); +} + /* Like RedisModule_CreatString(), but creates a string starting from a long * double. * @@ -7656,6 +7667,7 @@ void moduleRegisterCoreAPI(void) { REGISTER_API(CreateStringFromCallReply); REGISTER_API(CreateString); REGISTER_API(CreateStringFromLongLong); + REGISTER_API(CreateStringFromDouble); REGISTER_API(CreateStringFromLongDouble); REGISTER_API(CreateStringFromString); REGISTER_API(CreateStringPrintf); diff --git a/src/redismodule.h b/src/redismodule.h index 637078f2b..95dd80e8e 100644 --- a/src/redismodule.h +++ b/src/redismodule.h @@ -467,6 +467,7 @@ size_t REDISMODULE_API_FUNC(RedisModule_CallReplyLength)(RedisModuleCallReply *r RedisModuleCallReply *REDISMODULE_API_FUNC(RedisModule_CallReplyArrayElement)(RedisModuleCallReply *reply, size_t idx); RedisModuleString *REDISMODULE_API_FUNC(RedisModule_CreateString)(RedisModuleCtx *ctx, const char *ptr, size_t len); RedisModuleString *REDISMODULE_API_FUNC(RedisModule_CreateStringFromLongLong)(RedisModuleCtx *ctx, long long ll); +RedisModuleString *REDISMODULE_API_FUNC(RedisModule_CreateStringFromDouble)(RedisModuleCtx *ctx, double d); RedisModuleString *REDISMODULE_API_FUNC(RedisModule_CreateStringFromLongDouble)(RedisModuleCtx *ctx, long double ld, int humanfriendly); RedisModuleString *REDISMODULE_API_FUNC(RedisModule_CreateStringFromString)(RedisModuleCtx *ctx, const RedisModuleString *str); RedisModuleString *REDISMODULE_API_FUNC(RedisModule_CreateStringPrintf)(RedisModuleCtx *ctx, const char *fmt, ...); @@ -726,6 +727,7 @@ static int RedisModule_Init(RedisModuleCtx *ctx, const char *name, int ver, int REDISMODULE_GET_API(CreateStringFromCallReply); REDISMODULE_GET_API(CreateString); REDISMODULE_GET_API(CreateStringFromLongLong); + REDISMODULE_GET_API(CreateStringFromDouble); REDISMODULE_GET_API(CreateStringFromLongDouble); REDISMODULE_GET_API(CreateStringFromString); REDISMODULE_GET_API(CreateStringPrintf); From f7067d371bccb535c0b59907e24be05ab2b722cb Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Wed, 5 Feb 2020 09:42:49 +0200 Subject: [PATCH 0073/1280] memoryGetKeys helper function so that ACL can limit access to keys for MEMORY command --- src/db.c | 16 ++++++++++++++++ src/server.c | 2 +- src/server.h | 1 + 3 files changed, 18 insertions(+), 1 deletion(-) diff --git a/src/db.c b/src/db.c index ba7be2725..88342ac4d 100644 --- a/src/db.c +++ b/src/db.c @@ -1522,6 +1522,22 @@ int *georadiusGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numk return keys; } +/* Helper function to extract keys from memory command. + * MEMORY USAGE */ +int *memoryGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numkeys) { + int *keys; + UNUSED(cmd); + + if (argc >= 3 && !strcasecmp(argv[1]->ptr,"usage")) { + keys = zmalloc(sizeof(int) * 1); + keys[0] = 2; + *numkeys = 1; + return keys; + } + *numkeys = 0; + return NULL; +} + /* XREAD [BLOCK ] [COUNT ] [GROUP ] * STREAMS key_1 key_2 ... key_N ID_1 ID_2 ... ID_N */ int *xreadGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numkeys) { diff --git a/src/server.c b/src/server.c index 2b226f568..f3b7c37c5 100644 --- a/src/server.c +++ b/src/server.c @@ -817,7 +817,7 @@ struct redisCommand redisCommandTable[] = { {"memory",memoryCommand,-2, "random read-only", - 0,NULL,0,0,0,0,0,0}, + 0,memoryGetKeys,0,0,0,0,0,0}, {"client",clientCommand,-2, "admin no-script random @connection", diff --git a/src/server.h b/src/server.h index 5d63e9b55..9a403a946 100644 --- a/src/server.h +++ b/src/server.h @@ -2074,6 +2074,7 @@ int *sortGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numkeys); int *migrateGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numkeys); int *georadiusGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numkeys); int *xreadGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numkeys); +int *memoryGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numkeys); /* Cluster */ void clusterInit(void); From 097a0a52d6b59b3d8129084aa32c787e45307c52 Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Wed, 5 Feb 2020 11:41:24 +0200 Subject: [PATCH 0074/1280] config.c verbose error replies for CONFIG SET, like config file parsing We noticed that the error replies for the generic mechanism for enums are very verbose for config file parsing, but not for config set command. instead of replicating this code, i did a small refactoring to share code between CONFIG SET and config file parsing. and also renamed the enum group functions to be consistent with the naming of other types. --- src/config.c | 128 +++++++++++++-------------------------------------- 1 file changed, 31 insertions(+), 97 deletions(-) diff --git a/src/config.c b/src/config.c index 0526de84d..b19e78f74 100644 --- a/src/config.c +++ b/src/config.c @@ -190,8 +190,9 @@ typedef struct typeInterface { void (*init)(typeData data); /* Called on server start, should return 1 on success, 0 on error and should set err */ int (*load)(typeData data, sds *argc, int argv, char **err); - /* Called on CONFIG SET, returns 1 on success, 0 on error */ - int (*set)(typeData data, sds value, char **err); + /* Called on server startup and CONFIG SET, returns 1 on success, 0 on error + * and can set a verbose err string, update is true when called from CONFIG SET */ + int (*set)(typeData data, sds value, int update, char **err); /* Called on CONFIG GET, required to add output to the client */ void (*get)(client *c, typeData data); /* Called on CONFIG REWRITE, required to rewrite the config state */ @@ -323,7 +324,11 @@ void loadServerConfigFromString(char *config) { if ((!strcasecmp(argv[0],config->name) || (config->alias && !strcasecmp(argv[0],config->alias)))) { - if (!config->interface.load(config->data, argv, argc, &err)) { + if (argc != 2) { + err = "wrong number of arguments"; + goto loaderr; + } + if (!config->interface.set(config->data, argv[1], 0, &err)) { goto loaderr; } @@ -599,7 +604,7 @@ void configSetCommand(client *c) { if(config->modifiable && (!strcasecmp(c->argv[2]->ptr,config->name) || (config->alias && !strcasecmp(c->argv[2]->ptr,config->alias)))) { - if (!config->interface.set(config->data,o->ptr, &errstr)) { + if (!config->interface.set(config->data,o->ptr,1,&errstr)) { goto badfmt; } addReply(c,shared.ok); @@ -1536,9 +1541,8 @@ static char loadbuf[LOADBUF_SIZE]; .alias = (config_alias), \ .modifiable = (is_modifiable), -#define embedConfigInterface(initfn, loadfn, setfn, getfn, rewritefn) .interface = { \ +#define embedConfigInterface(initfn, setfn, getfn, rewritefn) .interface = { \ .init = (initfn), \ - .load = (loadfn), \ .set = (setfn), \ .get = (getfn), \ .rewrite = (rewritefn) \ @@ -1561,30 +1565,17 @@ static void boolConfigInit(typeData data) { *data.yesno.config = data.yesno.default_value; } -static int boolConfigLoad(typeData data, sds *argv, int argc, char **err) { - int yn; - if (argc != 2) { - *err = "wrong number of arguments"; - return 0; - } - if ((yn = yesnotoi(argv[1])) == -1) { +static int boolConfigSet(typeData data, sds value, int update, char **err) { + int yn = yesnotoi(value); + if (yn == -1) { *err = "argument must be 'yes' or 'no'"; return 0; } - if (data.yesno.is_valid_fn && !data.yesno.is_valid_fn(yn, err)) - return 0; - *data.yesno.config = yn; - return 1; -} - -static int boolConfigSet(typeData data, sds value, char **err) { - int yn = yesnotoi(value); - if (yn == -1) return 0; if (data.yesno.is_valid_fn && !data.yesno.is_valid_fn(yn, err)) return 0; int prev = *(data.yesno.config); *(data.yesno.config) = yn; - if (data.yesno.update_fn && !data.yesno.update_fn(yn, prev, err)) { + if (update && data.yesno.update_fn && !data.yesno.update_fn(yn, prev, err)) { *(data.yesno.config) = prev; return 0; } @@ -1601,7 +1592,7 @@ static void boolConfigRewrite(typeData data, const char *name, struct rewriteCon #define createBoolConfig(name, alias, modifiable, config_addr, default, is_valid, update) { \ embedCommonConfig(name, alias, modifiable) \ - embedConfigInterface(boolConfigInit, boolConfigLoad, boolConfigSet, boolConfigGet, boolConfigRewrite) \ + embedConfigInterface(boolConfigInit, boolConfigSet, boolConfigGet, boolConfigRewrite) \ .data.yesno = { \ .config = &(config_addr), \ .default_value = (default), \ @@ -1619,23 +1610,7 @@ static void stringConfigInit(typeData data) { } } -static int stringConfigLoad(typeData data, sds *argv, int argc, char **err) { - if (argc != 2) { - *err = "wrong number of arguments"; - return 0; - } - if (data.string.is_valid_fn && !data.string.is_valid_fn(argv[1], err)) - return 0; - zfree(*data.string.config); - if (data.string.convert_empty_to_null) { - *data.string.config = argv[1][0] ? zstrdup(argv[1]) : NULL; - } else { - *data.string.config = zstrdup(argv[1]); - } - return 1; -} - -static int stringConfigSet(typeData data, sds value, char **err) { +static int stringConfigSet(typeData data, sds value, int update, char **err) { if (data.string.is_valid_fn && !data.string.is_valid_fn(value, err)) return 0; char *prev = *data.string.config; @@ -1644,7 +1619,7 @@ static int stringConfigSet(typeData data, sds value, char **err) { } else { *data.string.config = zstrdup(value); } - if (data.string.update_fn && !data.string.update_fn(*data.string.config, prev, err)) { + if (update && data.string.update_fn && !data.string.update_fn(*data.string.config, prev, err)) { zfree(*data.string.config); *data.string.config = prev; return 0; @@ -1666,7 +1641,7 @@ static void stringConfigRewrite(typeData data, const char *name, struct rewriteC #define createStringConfig(name, alias, modifiable, empty_to_null, config_addr, default, is_valid, update) { \ embedCommonConfig(name, alias, modifiable) \ - embedConfigInterface(stringConfigInit, stringConfigLoad, stringConfigSet, stringConfigGet, stringConfigRewrite) \ + embedConfigInterface(stringConfigInit, stringConfigSet, stringConfigGet, stringConfigRewrite) \ .data.string = { \ .config = &(config_addr), \ .default_value = (default), \ @@ -1677,17 +1652,12 @@ static void stringConfigRewrite(typeData data, const char *name, struct rewriteC } /* Enum configs */ -static void configEnumInit(typeData data) { +static void enumConfigInit(typeData data) { *data.enumd.config = data.enumd.default_value; } -static int configEnumLoad(typeData data, sds *argv, int argc, char **err) { - if (argc != 2) { - *err = "wrong number of arguments"; - return 0; - } - - int enumval = configEnumGetValue(data.enumd.enum_value, argv[1]); +static int enumConfigSet(typeData data, sds value, int update, char **err) { + int enumval = configEnumGetValue(data.enumd.enum_value, value); if (enumval == INT_MIN) { sds enumerr = sdsnew("argument must be one of the following: "); configEnum *enumNode = data.enumd.enum_value; @@ -1707,37 +1677,28 @@ static int configEnumLoad(typeData data, sds *argv, int argc, char **err) { *err = loadbuf; return 0; } - if (data.enumd.is_valid_fn && !data.enumd.is_valid_fn(enumval, err)) - return 0; - *(data.enumd.config) = enumval; - return 1; -} - -static int configEnumSet(typeData data, sds value, char **err) { - int enumval = configEnumGetValue(data.enumd.enum_value, value); - if (enumval == INT_MIN) return 0; if (data.enumd.is_valid_fn && !data.enumd.is_valid_fn(enumval, err)) return 0; int prev = *(data.enumd.config); *(data.enumd.config) = enumval; - if (data.enumd.update_fn && !data.enumd.update_fn(enumval, prev, err)) { + if (update && data.enumd.update_fn && !data.enumd.update_fn(enumval, prev, err)) { *(data.enumd.config) = prev; return 0; } return 1; } -static void configEnumGet(client *c, typeData data) { +static void enumConfigGet(client *c, typeData data) { addReplyBulkCString(c, configEnumGetNameOrUnknown(data.enumd.enum_value,*data.enumd.config)); } -static void configEnumRewrite(typeData data, const char *name, struct rewriteConfigState *state) { +static void enumConfigRewrite(typeData data, const char *name, struct rewriteConfigState *state) { rewriteConfigEnumOption(state, name,*(data.enumd.config), data.enumd.enum_value, data.enumd.default_value); } #define createEnumConfig(name, alias, modifiable, enum, config_addr, default, is_valid, update) { \ embedCommonConfig(name, alias, modifiable) \ - embedConfigInterface(configEnumInit, configEnumLoad, configEnumSet, configEnumGet, configEnumRewrite) \ + embedConfigInterface(enumConfigInit, enumConfigSet, enumConfigGet, enumConfigRewrite) \ .data.enumd = { \ .config = &(config_addr), \ .default_value = (default), \ @@ -1832,49 +1793,22 @@ static int numericBoundaryCheck(typeData data, long long ll, char **err) { return 1; } -static int numericConfigLoad(typeData data, sds *argv, int argc, char **err) { - long long ll; - - if (argc != 2) { - *err = "wrong number of arguments"; - return 0; - } - +static int numericConfigSet(typeData data, sds value, int update, char **err) { + long long ll, prev = 0; if (data.numeric.is_memory) { int memerr; - ll = memtoll(argv[1], &memerr); + ll = memtoll(value, &memerr); if (memerr || ll < 0) { *err = "argument must be a memory value"; return 0; } } else { - if (!string2ll(argv[1], sdslen(argv[1]),&ll)) { + if (!string2ll(value, sdslen(value),&ll)) { *err = "argument couldn't be parsed into an integer" ; return 0; } } - if (!numericBoundaryCheck(data, ll, err)) - return 0; - - if (data.numeric.is_valid_fn && !data.numeric.is_valid_fn(ll, err)) - return 0; - - SET_NUMERIC_TYPE(ll) - - return 1; -} - -static int numericConfigSet(typeData data, sds value, char **err) { - long long ll, prev = 0; - if (data.numeric.is_memory) { - int memerr; - ll = memtoll(value, &memerr); - if (memerr || ll < 0) return 0; - } else { - if (!string2ll(value, sdslen(value),&ll)) return 0; - } - if (!numericBoundaryCheck(data, ll, err)) return 0; @@ -1884,7 +1818,7 @@ static int numericConfigSet(typeData data, sds value, char **err) { GET_NUMERIC_TYPE(prev) SET_NUMERIC_TYPE(ll) - if (data.numeric.update_fn && !data.numeric.update_fn(ll, prev, err)) { + if (update && data.numeric.update_fn && !data.numeric.update_fn(ll, prev, err)) { SET_NUMERIC_TYPE(prev) return 0; } @@ -1918,7 +1852,7 @@ static void numericConfigRewrite(typeData data, const char *name, struct rewrite #define embedCommonNumericalConfig(name, alias, modifiable, lower, upper, config_addr, default, memory, is_valid, update) { \ embedCommonConfig(name, alias, modifiable) \ - embedConfigInterface(numericConfigInit, numericConfigLoad, numericConfigSet, numericConfigGet, numericConfigRewrite) \ + embedConfigInterface(numericConfigInit, numericConfigSet, numericConfigGet, numericConfigRewrite) \ .data.numeric = { \ .lower_bound = (lower), \ .upper_bound = (upper), \ From 7d12ef55ca6a6b9b4bae63a91cf5035b2a439e9d Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Wed, 5 Feb 2020 18:06:33 +0200 Subject: [PATCH 0075/1280] Optimize temporary memory allocations for getKeysFromCommand mechanism now that we may use it more often (ACL), these excessive calls to malloc and free can become an overhead. --- src/db.c | 41 +++++++++++++++++++++++++++++++---------- 1 file changed, 31 insertions(+), 10 deletions(-) diff --git a/src/db.c b/src/db.c index 88342ac4d..caabeaabb 100644 --- a/src/db.c +++ b/src/db.c @@ -1292,6 +1292,8 @@ int expireIfNeeded(redisDb *db, robj *key) { /* ----------------------------------------------------------------------------- * API to get key arguments from commands * ---------------------------------------------------------------------------*/ +#define MAX_KEYS_BUFFER 65536 +static int getKeysTempBuffer[MAX_KEYS_BUFFER]; /* The base case is to use the keys position as given in the command table * (firstkey, lastkey, step). */ @@ -1306,7 +1308,12 @@ int *getKeysUsingCommandTable(struct redisCommand *cmd,robj **argv, int argc, in last = cmd->lastkey; if (last < 0) last = argc+last; - keys = zmalloc(sizeof(int)*((last - cmd->firstkey)+1)); + + int count = ((last - cmd->firstkey)+1); + keys = getKeysTempBuffer; + if (count > MAX_KEYS_BUFFER) + keys = zmalloc(sizeof(int)*count); + for (j = cmd->firstkey; j <= last; j += cmd->keystep) { if (j >= argc) { /* Modules commands, and standard commands with a not fixed number @@ -1316,7 +1323,7 @@ int *getKeysUsingCommandTable(struct redisCommand *cmd,robj **argv, int argc, in * return no keys and expect the command implementation to report * an arity or syntax error. */ if (cmd->flags & CMD_MODULE || cmd->arity < 0) { - zfree(keys); + getKeysFreeResult(keys); *numkeys = 0; return NULL; } else { @@ -1352,7 +1359,8 @@ int *getKeysFromCommand(struct redisCommand *cmd, robj **argv, int argc, int *nu /* Free the result of getKeysFromCommand. */ void getKeysFreeResult(int *result) { - zfree(result); + if (result != getKeysTempBuffer) + zfree(result); } /* Helper function to extract keys from following commands: @@ -1373,7 +1381,9 @@ int *zunionInterGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *nu /* Keys in z{union,inter}store come from two places: * argv[1] = storage key, * argv[3...n] = keys to intersect */ - keys = zmalloc(sizeof(int)*(num+1)); + keys = getKeysTempBuffer; + if (num+1>MAX_KEYS_BUFFER) + keys = zmalloc(sizeof(int)*(num+1)); /* Add all key positions for argv[3...n] to keys[] */ for (i = 0; i < num; i++) keys[i] = 3+i; @@ -1399,7 +1409,10 @@ int *evalGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numkeys) return NULL; } - keys = zmalloc(sizeof(int)*num); + keys = getKeysTempBuffer; + if (num>MAX_KEYS_BUFFER) + keys = zmalloc(sizeof(int)*num); + *numkeys = num; /* Add all key positions for argv[3...n] to keys[] */ @@ -1420,7 +1433,7 @@ int *sortGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numkeys) UNUSED(cmd); num = 0; - keys = zmalloc(sizeof(int)*2); /* Alloc 2 places for the worst case. */ + keys = getKeysTempBuffer; /* Alloc 2 places for the worst case. */ keys[num++] = 1; /* is always present. */ @@ -1478,7 +1491,10 @@ int *migrateGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numkey } } - keys = zmalloc(sizeof(int)*num); + keys = getKeysTempBuffer; + if (num>MAX_KEYS_BUFFER) + keys = zmalloc(sizeof(int)*num); + for (i = 0; i < num; i++) keys[i] = first+i; *numkeys = num; return keys; @@ -1511,7 +1527,9 @@ int *georadiusGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numk * argv[1] = key, * argv[5...n] = stored key if present */ - keys = zmalloc(sizeof(int) * num); + keys = getKeysTempBuffer; + if (num>MAX_KEYS_BUFFER) + keys = zmalloc(sizeof(int) * num); /* Add all key positions to keys[] */ keys[0] = 1; @@ -1529,7 +1547,7 @@ int *memoryGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numkeys UNUSED(cmd); if (argc >= 3 && !strcasecmp(argv[1]->ptr,"usage")) { - keys = zmalloc(sizeof(int) * 1); + keys = getKeysTempBuffer; keys[0] = 2; *numkeys = 1; return keys; @@ -1576,7 +1594,10 @@ int *xreadGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numkeys) num /= 2; /* We have half the keys as there are arguments because there are also the IDs, one per key. */ - keys = zmalloc(sizeof(int) * num); + keys = getKeysTempBuffer; + if (num>MAX_KEYS_BUFFER) + keys = zmalloc(sizeof(int) * num); + for (i = streams_pos+1; i < argc-num; i++) keys[i-streams_pos-1] = i; *numkeys = num; return keys; From 13a737902469104b535cb19d1c6e5f011aba81b4 Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Wed, 5 Feb 2020 18:15:38 +0200 Subject: [PATCH 0076/1280] update RM_SignalModifiedKey doc comment --- src/module.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/module.c b/src/module.c index 914c50df3..97edb97b3 100644 --- a/src/module.c +++ b/src/module.c @@ -890,7 +890,8 @@ void RM_SetModuleOptions(RedisModuleCtx *ctx, int options) { ctx->module->options = options; } -/* Signals that the key is modified from user's perspective (i.e. invalidate WATCH). */ +/* Signals that the key is modified from user's perspective (i.e. invalidate WATCH + * and client side caching). */ int RM_SignalModifiedKey(RedisModuleCtx *ctx, RedisModuleString *keyname) { signalModifiedKey(ctx->client->db,keyname); return REDISMODULE_OK; From f4c7f1ac272904007ae9b5b845e51e83c68a0cfc Mon Sep 17 00:00:00 2001 From: Yossi Gottlieb Date: Wed, 5 Feb 2020 18:30:12 +0200 Subject: [PATCH 0077/1280] TLS: Some redis.conf clarifications. --- redis.conf | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/redis.conf b/redis.conf index 07005cffe..3c7336747 100644 --- a/redis.conf +++ b/redis.conf @@ -155,23 +155,22 @@ tcp-keepalive 300 # tls-ca-cert-file ca.crt # tls-ca-cert-dir /etc/ssl/certs -# If TLS/SSL clients are required to authenticate using a client side -# certificate, use this directive. +# By default, clients (including replica servers) on a TLS port are required +# to authenticate using valid client side certificates. # -# Note: this applies to all incoming clients, including replicas. +# It is possible to disable authentication using this directive. # -# tls-auth-clients yes +# tls-auth-clients no -# If TLS/SSL should be used when connecting as a replica to a master, enable -# this configuration directive: +# By default, a Redis replica does not attempt to establish a TLS connection +# with its master. +# +# Use the following directive to enable TLS on replication links. # # tls-replication yes -# If TLS/SSL should be used for the Redis Cluster bus, enable this configuration -# directive. -# -# NOTE: If TLS/SSL is enabled for Cluster Bus, mutual authentication is always -# enforced. +# By default, the Redis Cluster bus uses a plain TCP connection. To enable +# TLS for the bus protocol, use the following directive: # # tls-cluster yes From 467090e45f5928762a6e3b1e20414021dc3edbac Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Wed, 5 Feb 2020 19:47:09 +0200 Subject: [PATCH 0078/1280] Add handling of short read of module id in rdb --- src/rdb.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/rdb.c b/src/rdb.c index 27e1b3135..61265433d 100644 --- a/src/rdb.c +++ b/src/rdb.c @@ -1871,7 +1871,10 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, robj *key) { } } else if (rdbtype == RDB_TYPE_MODULE || rdbtype == RDB_TYPE_MODULE_2) { uint64_t moduleid = rdbLoadLen(rdb,NULL); - if (rioGetReadError(rdb)) return NULL; + if (rioGetReadError(rdb)) { + rdbReportReadError("Short read module id"); + return NULL; + } moduleType *mt = moduleTypeLookupModuleByID(moduleid); char name[10]; From bc5e0a456b6786a01dc71387a17eb9d2a3d1a3f7 Mon Sep 17 00:00:00 2001 From: Yossi Gottlieb Date: Wed, 5 Feb 2020 21:13:21 +0200 Subject: [PATCH 0079/1280] TLS: Update documentation. --- README.md | 18 ++++++++++++++++++ TLS.md | 45 ++++++++++++++------------------------------- 2 files changed, 32 insertions(+), 31 deletions(-) diff --git a/README.md b/README.md index 3442659e6..c08013416 100644 --- a/README.md +++ b/README.md @@ -35,6 +35,11 @@ It is as simple as: % make +To build with TLS support, you'll need OpenSSL development libraries (e.g. +libssl-dev on Debian/Ubuntu) and run: + + % make BUILD_TLS=yes + You can run a 32 bit Redis binary using: % make 32bit @@ -43,6 +48,13 @@ After building Redis, it is a good idea to test it using: % make test +If TLS is built, running the tests with TLS enabled (you will need `tcl-tls` +installed): + + % ./utils/gen-test-certs.sh + % ./runtest --tls + + Fixing build problems with dependencies or cached build options --------- @@ -125,6 +137,12 @@ as options using the command line. Examples: All the options in redis.conf are also supported as options using the command line, with exactly the same name. +Running Redis with TLS: +------------------ + +Please consult the [TLS.md](TLS.md) file for more information on +how to use Redis with TLS. + Playing with Redis ------------------ diff --git a/TLS.md b/TLS.md index 76fe0be2e..e480c1e9d 100644 --- a/TLS.md +++ b/TLS.md @@ -1,8 +1,5 @@ -TLS Support -- Work In Progress -=============================== - -This is a brief note to capture current thoughts/ideas and track pending action -items. +TLS Support +=========== Getting Started --------------- @@ -69,37 +66,23 @@ probably not be so hard. For cluster keys migration it might be more difficult, but there are probably other good reasons to improve that part anyway. To-Do List -========== +---------- -Additional TLS Features ------------------------ +- [ ] Add session caching support. Check if/how it's handled by clients to + assess how useful/important it is. +- [ ] redis-benchmark support. The current implementation is a mix of using + hiredis for parsing and basic networking (establishing connections), but + directly manipulating sockets for most actions. This will need to be cleaned + up for proper TLS support. The best approach is probably to migrate to hiredis + async mode. +- [ ] redis-cli `--slave` and `--rdb` support. -1. Add metrics to INFO? -2. Add session caching support. Check if/how it's handled by clients to assess - how useful/important it is. - -redis-benchmark ---------------- - -The current implementation is a mix of using hiredis for parsing and basic -networking (establishing connections), but directly manipulating sockets for -most actions. - -This will need to be cleaned up for proper TLS support. The best approach is -probably to migrate to hiredis async mode. - -redis-cli ---------- - -1. Add support for TLS in --slave and --rdb modes. - -Others ------- +Multi-port +---------- Consider the implications of allowing TLS to be configured on a separate port, -making Redis listening on multiple ports. +making Redis listening on multiple ports: -This impacts many things, like 1. Startup banner port notification 2. Proctitle 3. How slaves announce themselves From ddc18452867055255575839e5cb560d262d711c4 Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Thu, 6 Feb 2020 08:53:23 +0200 Subject: [PATCH 0080/1280] A few non-data commands that should be allowed while loading or stale SELECT, and HELLO are commands that may be executed by the client as soon as it connects, there's no reason to block them, preventing the client from doing the rest of his sequence (which might just be INFO or CONFIG, etc). MONITOR, DEBUG, SLOWLOG, TIME, LASTSAVE are all non-data accessing commands, which there's no reason to block. --- src/server.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/server.c b/src/server.c index f3b7c37c5..23e72ce19 100644 --- a/src/server.c +++ b/src/server.c @@ -579,7 +579,7 @@ struct redisCommand redisCommandTable[] = { 0,NULL,0,0,0,0,0,0}, {"select",selectCommand,2, - "ok-loading fast @keyspace", + "ok-loading fast ok-stale @keyspace", 0,NULL,0,0,0,0,0,0}, {"swapdb",swapdbCommand,3, @@ -660,7 +660,7 @@ struct redisCommand redisCommandTable[] = { 0,NULL,0,0,0,0,0,0}, {"lastsave",lastsaveCommand,1, - "read-only random fast @admin @dangerous", + "read-only random fast ok-loading ok-stale @admin @dangerous", 0,NULL,0,0,0,0,0,0}, {"type",typeCommand,2, @@ -708,7 +708,7 @@ struct redisCommand redisCommandTable[] = { 0,NULL,0,0,0,0,0,0}, {"monitor",monitorCommand,1, - "admin no-script", + "admin no-script ok-loading ok-stale", 0,NULL,0,0,0,0,0,0}, {"ttl",ttlCommand,2, @@ -740,7 +740,7 @@ struct redisCommand redisCommandTable[] = { 0,NULL,0,0,0,0,0,0}, {"debug",debugCommand,-2, - "admin no-script", + "admin no-script ok-loading ok-stale", 0,NULL,0,0,0,0,0,0}, {"config",configCommand,-2, @@ -820,11 +820,11 @@ struct redisCommand redisCommandTable[] = { 0,memoryGetKeys,0,0,0,0,0,0}, {"client",clientCommand,-2, - "admin no-script random @connection", + "admin no-script random ok-loading ok-stale @connection", 0,NULL,0,0,0,0,0,0}, {"hello",helloCommand,-2, - "no-auth no-script fast no-monitor no-slowlog @connection", + "no-auth no-script fast no-monitor ok-loading ok-stale no-slowlog @connection", 0,NULL,0,0,0,0,0,0}, /* EVAL can modify the dataset, however it is not flagged as a write @@ -838,7 +838,7 @@ struct redisCommand redisCommandTable[] = { 0,evalGetKeys,0,0,0,0,0,0}, {"slowlog",slowlogCommand,-2, - "admin random", + "admin random ok-loading ok-stale", 0,NULL,0,0,0,0,0,0}, {"script",scriptCommand,-2, @@ -846,7 +846,7 @@ struct redisCommand redisCommandTable[] = { 0,NULL,0,0,0,0,0,0}, {"time",timeCommand,1, - "read-only random fast", + "read-only random fast ok-loading ok-stale", 0,NULL,0,0,0,0,0,0}, {"bitop",bitopCommand,-4, From f7bb301644ab1414ed5e36c1be7c16413c697cff Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Thu, 6 Feb 2020 09:15:31 +0200 Subject: [PATCH 0081/1280] Memory leak when bind config is provided twice --- src/config.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/config.c b/src/config.c index b19e78f74..fe26b3884 100644 --- a/src/config.c +++ b/src/config.c @@ -349,6 +349,10 @@ void loadServerConfigFromString(char *config) { if (addresses > CONFIG_BINDADDR_MAX) { err = "Too many bind addresses specified"; goto loaderr; } + /* Free old bind addresses */ + for (j = 0; j < server.bindaddr_count; j++) { + zfree(server.bindaddr[j]); + } for (j = 0; j < addresses; j++) server.bindaddr[j] = zstrdup(argv[j+1]); server.bindaddr_count = addresses; From e41c1824d6fc05032d1a500c0e6171a400f21708 Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Thu, 6 Feb 2020 09:23:22 +0200 Subject: [PATCH 0082/1280] fix maxmemory config warning the warning condition was if usage > limit (saying it'll cause eviction or oom), but in fact the eviction and oom depends on used minus slave buffers. other than fixing the condition, i add info about the current usage and limit, which may be useful when looking at the log. --- src/config.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/config.c b/src/config.c index b19e78f74..173a90909 100644 --- a/src/config.c +++ b/src/config.c @@ -1995,8 +1995,9 @@ static int updateMaxmemory(long long val, long long prev, char **err) { UNUSED(prev); UNUSED(err); if (val) { - if ((unsigned long long)val < zmalloc_used_memory()) { - serverLog(LL_WARNING,"WARNING: the new maxmemory value set via CONFIG SET is smaller than the current memory usage. This will result in key eviction and/or the inability to accept new write commands depending on the maxmemory-policy."); + size_t used = zmalloc_used_memory()-freeMemoryGetNotCountedMemory(); + if ((unsigned long long)val < used) { + serverLog(LL_WARNING,"WARNING: the new maxmemory value set via CONFIG SET (%llu) is smaller than the current memory usage (%zu). This will result in key eviction and/or the inability to accept new write commands depending on the maxmemory-policy.", server.maxmemory, used); } freeMemoryIfNeededAndSafe(); } From 4050c57139cfe7e82a7173b0cd9d1bf5a497c829 Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Thu, 6 Feb 2020 09:33:20 +0200 Subject: [PATCH 0083/1280] Fix client flags to be int64 in module.c currently there's no bug since the flags these functions handle are always lower than 32bit, but still better fix the type to prevent future bugs. --- src/module.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/module.c b/src/module.c index 914c50df3..a2bc869f9 100644 --- a/src/module.c +++ b/src/module.c @@ -714,9 +714,9 @@ void RM_KeyAtPos(RedisModuleCtx *ctx, int pos) { * flags into the command flags used by the Redis core. * * It returns the set of flags, or -1 if unknown flags are found. */ -int commandFlagsFromString(char *s) { +int64_t commandFlagsFromString(char *s) { int count, j; - int flags = 0; + int64_t flags = 0; sds *tokens = sdssplitlen(s,strlen(s)," ",1,&count); for (j = 0; j < count; j++) { char *t = tokens[j]; @@ -798,7 +798,7 @@ int commandFlagsFromString(char *s) { * to authenticate a client. */ int RM_CreateCommand(RedisModuleCtx *ctx, const char *name, RedisModuleCmdFunc cmdfunc, const char *strflags, int firstkey, int lastkey, int keystep) { - int flags = strflags ? commandFlagsFromString((char*)strflags) : 0; + int64_t flags = strflags ? commandFlagsFromString((char*)strflags) : 0; if (flags == -1) return REDISMODULE_ERR; if ((flags & CMD_MODULE_NO_CLUSTER) && server.cluster_enabled) return REDISMODULE_ERR; From 01eadbd6e1fd98fdbc4f1adaddcf3851735f5748 Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Thu, 6 Feb 2020 09:37:04 +0200 Subject: [PATCH 0084/1280] moduleRDBLoadError, add key name, and use panic rather than exit using panic rather than exit means you get s stack trace of the code path that experianced the error, and possibly other info. --- src/module.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/module.c b/src/module.c index 914c50df3..16b191535 100644 --- a/src/module.c +++ b/src/module.c @@ -3649,14 +3649,15 @@ void moduleRDBLoadError(RedisModuleIO *io) { io->error = 1; return; } - serverLog(LL_WARNING, + serverPanic( "Error loading data from RDB (short read or EOF). " "Read performed by module '%s' about type '%s' " - "after reading '%llu' bytes of a value.", + "after reading '%llu' bytes of a value " + "for key named: '%s'.", io->type->module->name, io->type->name, - (unsigned long long)io->bytes); - exit(1); + (unsigned long long)io->bytes, + io->key? (char*)io->key->ptr: "(null)"); } /* Returns 0 if there's at least one registered data type that did not declare From 3ea0716cad0ad73b76527d85e68b4b3261874d49 Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Thu, 6 Feb 2020 09:41:45 +0200 Subject: [PATCH 0085/1280] reduce repeated calls to use_diskless_load this function possibly iterates on the module list --- src/replication.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/replication.c b/src/replication.c index b7e77184a..7aebd620a 100644 --- a/src/replication.c +++ b/src/replication.c @@ -1354,7 +1354,7 @@ void disklessLoadRestoreBackups(redisDb *backup, int restore, int empty_db_flags void readSyncBulkPayload(connection *conn) { char buf[4096]; ssize_t nread, readlen, nwritten; - int use_diskless_load; + int use_diskless_load = useDisklessLoad(); redisDb *diskless_load_backup = NULL; int empty_db_flags = server.repl_slave_lazy_flush ? EMPTYDB_ASYNC : EMPTYDB_NO_FLAGS; @@ -1411,19 +1411,18 @@ void readSyncBulkPayload(connection *conn) { server.repl_transfer_size = 0; serverLog(LL_NOTICE, "MASTER <-> REPLICA sync: receiving streamed RDB from master with EOF %s", - useDisklessLoad()? "to parser":"to disk"); + use_diskless_load? "to parser":"to disk"); } else { usemark = 0; server.repl_transfer_size = strtol(buf+1,NULL,10); serverLog(LL_NOTICE, "MASTER <-> REPLICA sync: receiving %lld bytes from master %s", (long long) server.repl_transfer_size, - useDisklessLoad()? "to parser":"to disk"); + use_diskless_load? "to parser":"to disk"); } return; } - use_diskless_load = useDisklessLoad(); if (!use_diskless_load) { /* Read the data from the socket, store it to a file and search * for the EOF. */ From df866ad32214e5c8b3d3221424ff43a29d539698 Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Thu, 6 Feb 2020 10:07:17 +0200 Subject: [PATCH 0086/1280] freeClientAsync don't lock mutex if there's just one thread --- src/networking.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/networking.c b/src/networking.c index a2e454d4b..1c9325c43 100644 --- a/src/networking.c +++ b/src/networking.c @@ -1162,9 +1162,14 @@ void freeClientAsync(client *c) { * may access the list while Redis uses I/O threads. All the other accesses * are in the context of the main thread while the other threads are * idle. */ - static pthread_mutex_t async_free_queue_mutex = PTHREAD_MUTEX_INITIALIZER; if (c->flags & CLIENT_CLOSE_ASAP || c->flags & CLIENT_LUA) return; c->flags |= CLIENT_CLOSE_ASAP; + if (server.io_threads_num == 1) { + /* no need to bother with locking if there's just one thread (the main thread) */ + listAddNodeTail(server.clients_to_close,c); + return; + } + static pthread_mutex_t async_free_queue_mutex = PTHREAD_MUTEX_INITIALIZER; pthread_mutex_lock(&async_free_queue_mutex); listAddNodeTail(server.clients_to_close,c); pthread_mutex_unlock(&async_free_queue_mutex); From 3a74aa3ebe3c3cb3020d80660f39d662d68020b6 Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Thu, 6 Feb 2020 10:14:32 +0200 Subject: [PATCH 0087/1280] move restartAOFAfterSYNC from replicaofCommand to replicationUnsetMaster replicationUnsetMaster can be called from other places, not just replicaofCOmmand, and all of these need to restart AOF --- src/replication.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/replication.c b/src/replication.c index b7e77184a..5499ebc57 100644 --- a/src/replication.c +++ b/src/replication.c @@ -2399,6 +2399,10 @@ void replicationUnsetMaster(void) { moduleFireServerEvent(REDISMODULE_EVENT_REPLICATION_ROLE_CHANGED, REDISMODULE_EVENT_REPLROLECHANGED_NOW_MASTER, NULL); + + /* Restart the AOF subsystem in case we shut it down during a sync when + * we were still a slave. */ + if (server.aof_enabled && server.aof_state == AOF_OFF) restartAOFAfterSYNC(); } /* This function is called when the slave lose the connection with the @@ -2436,9 +2440,6 @@ void replicaofCommand(client *c) { serverLog(LL_NOTICE,"MASTER MODE enabled (user request from '%s')", client); sdsfree(client); - /* Restart the AOF subsystem in case we shut it down during a sync when - * we were still a slave. */ - if (server.aof_enabled && server.aof_state == AOF_OFF) restartAOFAfterSYNC(); } } else { long port; From a772eb89cc765f6e957b1b288101332af48aef07 Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Thu, 6 Feb 2020 10:17:34 +0200 Subject: [PATCH 0088/1280] stopAppendOnly resets aof_rewrite_scheduled althouh in theory, users can do BGREWRITEAOF even if aof is disabled, i suppose it is more common that the scheduled flag is set by either startAppendOnly, of a failed initial AOFRW fork (AOF_WAIT_REWRITE) --- src/aof.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/aof.c b/src/aof.c index 9eeb3f1e2..3682c4568 100644 --- a/src/aof.c +++ b/src/aof.c @@ -242,6 +242,7 @@ void stopAppendOnly(void) { server.aof_fd = -1; server.aof_selected_db = -1; server.aof_state = AOF_OFF; + server.aof_rewrite_scheduled = 0; killAppendOnlyChild(); } From 7e53f26984e64849880b7268f0296600b6f5ba19 Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Thu, 6 Feb 2020 10:31:43 +0200 Subject: [PATCH 0089/1280] add SAVE subcommand to ACL HELP and top comment --- src/acl.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/acl.c b/src/acl.c index 1f395bd3f..aae35226f 100644 --- a/src/acl.c +++ b/src/acl.c @@ -1461,6 +1461,7 @@ void ACLLoadUsersAtStartup(void) { /* ACL -- show and modify the configuration of ACL users. * ACL HELP * ACL LOAD + * ACL SAVE * ACL LIST * ACL USERS * ACL CAT [] @@ -1658,6 +1659,7 @@ void aclCommand(client *c) { } else if (!strcasecmp(sub,"help")) { const char *help[] = { "LOAD -- Reload users from the ACL file.", +"SAVE -- Save the current config to the ACL file." "LIST -- Show user details in config file format.", "USERS -- List all the registered usernames.", "SETUSER [attribs ...] -- Create or modify a user.", From 818b77193bd322ae7a4916419bef72628c059146 Mon Sep 17 00:00:00 2001 From: Guy Benoish Date: Thu, 6 Feb 2020 14:09:45 +0530 Subject: [PATCH 0090/1280] Fix small bugs related to replica and monitor ambiguity 1. server.repl_no_slaves_since can be set when a MONITOR client disconnects 2. c->repl_ack_time can be set by a newline from a MONITOR client 3. Improved comments --- src/networking.c | 12 +++++++----- src/server.c | 2 +- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/networking.c b/src/networking.c index a2e454d4b..8fca3fc1d 100644 --- a/src/networking.c +++ b/src/networking.c @@ -1121,7 +1121,7 @@ void freeClient(client *c) { /* We need to remember the time when we started to have zero * attached slaves, as after some time we'll free the replication * backlog. */ - if (c->flags & CLIENT_SLAVE && listLength(server.slaves) == 0) + if (getClientType(c) == CLIENT_TYPE_SLAVE && listLength(server.slaves) == 0) server.repl_no_slaves_since = server.unixtime; refreshGoodSlavesCount(); /* Fire the replica change modules event. */ @@ -1252,8 +1252,8 @@ int writeToClient(client *c, int handler_installed) { * just deliver as much data as it is possible to deliver. * * Moreover, we also send as much as possible if the client is - * a slave (otherwise, on high-speed traffic, the replication - * buffer will grow indefinitely) */ + * a slave or a monitor (otherwise, on high-speed traffic, the + * replication/output buffer will grow indefinitely) */ if (totwritten > NET_MAX_WRITES_PER_EVENT && (server.maxmemory == 0 || zmalloc_used_memory() < server.maxmemory) && @@ -1439,7 +1439,7 @@ int processInlineBuffer(client *c) { /* Newline from slaves can be used to refresh the last ACK time. * This is useful for a slave to ping back while loading a big * RDB file. */ - if (querylen == 0 && c->flags & CLIENT_SLAVE) + if (querylen == 0 && getClientType(c) == CLIENT_TYPE_SLAVE) c->repl_ack_time = server.unixtime; /* Move querybuffer position to the next query in the buffer. */ @@ -2433,12 +2433,14 @@ unsigned long getClientOutputBufferMemoryUsage(client *c) { * * The function will return one of the following: * CLIENT_TYPE_NORMAL -> Normal client - * CLIENT_TYPE_SLAVE -> Slave or client executing MONITOR command + * CLIENT_TYPE_SLAVE -> Slave * CLIENT_TYPE_PUBSUB -> Client subscribed to Pub/Sub channels * CLIENT_TYPE_MASTER -> The client representing our replication master. */ int getClientType(client *c) { if (c->flags & CLIENT_MASTER) return CLIENT_TYPE_MASTER; + /* Even though MONITOR clients are marked as replicas, we + * want the expose them as normal clients. */ if ((c->flags & CLIENT_SLAVE) && !(c->flags & CLIENT_MONITOR)) return CLIENT_TYPE_SLAVE; if (c->flags & CLIENT_PUBSUB) return CLIENT_TYPE_PUBSUB; diff --git a/src/server.c b/src/server.c index f3b7c37c5..2b32df0db 100644 --- a/src/server.c +++ b/src/server.c @@ -1498,7 +1498,7 @@ int clientsCronHandleTimeout(client *c, mstime_t now_ms) { time_t now = now_ms/1000; if (server.maxidletime && - !(c->flags & CLIENT_SLAVE) && /* no timeout for slaves */ + !(c->flags & CLIENT_SLAVE) && /* no timeout for slaves and monitors */ !(c->flags & CLIENT_MASTER) && /* no timeout for masters */ !(c->flags & CLIENT_BLOCKED) && /* no timeout for BLPOP */ !(c->flags & CLIENT_PUBSUB) && /* no timeout for Pub/Sub clients */ From bc705b1b2624971dba4ade2ebb87eec921471d2c Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Thu, 6 Feb 2020 10:40:29 +0200 Subject: [PATCH 0091/1280] DEBUG HELP - add PROTOCOL --- src/debug.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/debug.c b/src/debug.c index a2d37337d..b910d2d2d 100644 --- a/src/debug.c +++ b/src/debug.c @@ -355,6 +355,7 @@ void debugCommand(client *c) { "CRASH-AND-RECOVER -- Hard crash and restart after delay.", "DIGEST -- Output a hex signature representing the current DB content.", "DIGEST-VALUE ... -- Output a hex signature of the values of all the specified keys.", +"DEBUG PROTOCOL [string|integer|double|bignum|null|array|set|map|attrib|push|verbatim|true|false]", "ERROR -- Return a Redis protocol error with as message. Useful for clients unit tests to simulate Redis errors.", "LOG -- write message to the server log.", "HTSTATS -- Return hash table statistics of the specified Redis database.", @@ -586,7 +587,7 @@ NULL } } else if (!strcasecmp(c->argv[1]->ptr,"protocol") && c->argc == 3) { /* DEBUG PROTOCOL [string|integer|double|bignum|null|array|set|map| - * attrib|push|verbatim|true|false|state|err|bloberr] */ + * attrib|push|verbatim|true|false] */ char *name = c->argv[2]->ptr; if (!strcasecmp(name,"string")) { addReplyBulkCString(c,"Hello World"); @@ -634,7 +635,7 @@ NULL } else if (!strcasecmp(name,"verbatim")) { addReplyVerbatim(c,"This is a verbatim\nstring",25,"txt"); } else { - addReplyError(c,"Wrong protocol type name. Please use one of the following: string|integer|double|bignum|null|array|set|map|attrib|push|verbatim|true|false|state|err|bloberr"); + addReplyError(c,"Wrong protocol type name. Please use one of the following: string|integer|double|bignum|null|array|set|map|attrib|push|verbatim|true|false"); } } else if (!strcasecmp(c->argv[1]->ptr,"sleep") && c->argc == 3) { double dtime = strtod(c->argv[2]->ptr,NULL); From 2b10c2ce47faf0e4fda887bd247de67795eb05c8 Mon Sep 17 00:00:00 2001 From: Guy Benoish Date: Thu, 6 Feb 2020 14:12:08 +0530 Subject: [PATCH 0092/1280] Some refactroing using getClientType instead of CLIENT_SLAVE --- src/networking.c | 9 +++++---- src/object.c | 35 +++++++++++++---------------------- 2 files changed, 18 insertions(+), 26 deletions(-) diff --git a/src/networking.c b/src/networking.c index 8fca3fc1d..496b0f3dc 100644 --- a/src/networking.c +++ b/src/networking.c @@ -369,9 +369,10 @@ void addReplyErrorLength(client *c, const char *s, size_t len) { * Where the master must propagate the first change even if the second * will produce an error. However it is useful to log such events since * they are rare and may hint at errors in a script or a bug in Redis. */ - if (c->flags & (CLIENT_MASTER|CLIENT_SLAVE) && !(c->flags & CLIENT_MONITOR)) { - char* to = c->flags & CLIENT_MASTER? "master": "replica"; - char* from = c->flags & CLIENT_MASTER? "replica": "master"; + int ctype = getClientType(c); + if (ctype == CLIENT_TYPE_MASTER || ctype == CLIENT_TYPE_SLAVE) { + char* to = ctype == CLIENT_TYPE_MASTER? "master": "replica"; + char* from = ctype == CLIENT_TYPE_MASTER? "replica": "master"; char *cmdname = c->lastcmd ? c->lastcmd->name : ""; serverLog(LL_WARNING,"== CRITICAL == This %s is sending an error " "to its %s: '%s' after processing the command " @@ -1074,7 +1075,7 @@ void freeClient(client *c) { } /* Log link disconnection with slave */ - if ((c->flags & CLIENT_SLAVE) && !(c->flags & CLIENT_MONITOR)) { + if (getClientType(c) == CLIENT_TYPE_SLAVE) { serverLog(LL_WARNING,"Connection with replica %s lost.", replicationGetSlaveName(c)); } diff --git a/src/object.c b/src/object.c index 52007b474..cc6b218a0 100644 --- a/src/object.c +++ b/src/object.c @@ -974,38 +974,29 @@ struct redisMemOverhead *getMemoryOverheadData(void) { mh->repl_backlog = mem; mem_total += mem; - mem = 0; - if (listLength(server.slaves)) { - listIter li; - listNode *ln; - - listRewind(server.slaves,&li); - while((ln = listNext(&li))) { - client *c = listNodeValue(ln); - mem += getClientOutputBufferMemoryUsage(c); - mem += sdsAllocSize(c->querybuf); - mem += sizeof(client); - } - } - mh->clients_slaves = mem; - mem_total+=mem; - mem = 0; if (listLength(server.clients)) { listIter li; listNode *ln; + size_t mem_normal = 0, mem_slaves = 0; listRewind(server.clients,&li); while((ln = listNext(&li))) { + size_t mem_curr = 0; client *c = listNodeValue(ln); - if (c->flags & CLIENT_SLAVE && !(c->flags & CLIENT_MONITOR)) - continue; - mem += getClientOutputBufferMemoryUsage(c); - mem += sdsAllocSize(c->querybuf); - mem += sizeof(client); + int type = getClientType(c); + mem_curr += getClientOutputBufferMemoryUsage(c); + mem_curr += sdsAllocSize(c->querybuf); + mem_curr += sizeof(client); + if (type == CLIENT_TYPE_SLAVE) + mem_slaves += mem_curr; + else + mem_normal += mem_curr; } + mh->clients_slaves = mem_slaves; + mh->clients_normal = mem_normal; + mem = mem_slaves + mem_normal; } - mh->clients_normal = mem; mem_total+=mem; mem = 0; From f390ad6d5136dd52dc154ec671eedbc6e1fb95dc Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Wed, 5 Feb 2020 18:24:14 +0200 Subject: [PATCH 0093/1280] RM_Scan disable dict rehashing The callback approach we took is very efficient, the module can do any filtering of keys without building any list and cloning strings, it can also read data from the key's value. but if the user tries to re-open the key, or any other key, this can cause dict re-hashing (dictFind does that), and that's very bad to do from inside dictScan. this commit protects the dict from doing any rehashing during scan, but also warns the user not to attempt any writes or command calls from within the callback, for fear of unexpected side effects and crashes. --- src/dict.c | 7 +++++++ src/module.c | 20 ++++++++++++++------ 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/src/dict.c b/src/dict.c index 106467ef7..93e6c39a7 100644 --- a/src/dict.c +++ b/src/dict.c @@ -871,6 +871,10 @@ unsigned long dictScan(dict *d, if (dictSize(d) == 0) return 0; + /* Having a safe iterator means no rehashing can happen, see _dictRehashStep. + * This is needed in case the scan callback tries to do dictFind or alike. */ + d->iterators++; + if (!dictIsRehashing(d)) { t0 = &(d->ht[0]); m0 = t0->sizemask; @@ -937,6 +941,9 @@ unsigned long dictScan(dict *d, } while (v & (m0 ^ m1)); } + /* undo the ++ at the top */ + d->iterators--; + return v; } diff --git a/src/module.c b/src/module.c index 914c50df3..e9bceac47 100644 --- a/src/module.c +++ b/src/module.c @@ -6515,9 +6515,13 @@ void RM_ScanCursorDestroy(RedisModuleScanCursor *cursor) { * } * RedisModule_ScanCursorDestroy(c); * - * The function will return 1 if there are more elements to scan and 0 otherwise, - * possibly setting errno if the call failed. - * It is also possible to restart and existing cursor using RM_CursorRestart. */ + * The function will return 1 if there are more elements to scan and 0 otherwise, + * possibly setting errno if the call failed. + * It is also possible to restart and existing cursor using RM_CursorRestart. + * + * NOTE: You must avoid doing any database changes from within the callback, you should avoid any + * RedisModule_OpenKey or RedisModule_Call, if you need to do these, you need to keep the key name + * and do any work you need to do after the call to Scan returns. */ int RM_Scan(RedisModuleCtx *ctx, RedisModuleScanCursor *cursor, RedisModuleScanCB fn, void *privdata) { if (cursor->done) { errno = ENOENT; @@ -6595,9 +6599,13 @@ static void moduleScanKeyCallback(void *privdata, const dictEntry *de) { * RedisModule_CloseKey(key); * RedisModule_ScanCursorDestroy(c); * - * The function will return 1 if there are more elements to scan and 0 otherwise, - * possibly setting errno if the call failed. - * It is also possible to restart and existing cursor using RM_CursorRestart. */ + * The function will return 1 if there are more elements to scan and 0 otherwise, + * possibly setting errno if the call failed. + * It is also possible to restart and existing cursor using RM_CursorRestart. + * + * NOTE: You must avoid doing any database changes from within the callback, you should avoid any + * RedisModule_OpenKey or RedisModule_Call, if you need to do these, you need to keep the field name + * and do any work you need to do after the call to Scan returns. */ int RM_ScanKey(RedisModuleKey *key, RedisModuleScanCursor *cursor, RedisModuleScanKeyCB fn, void *privdata) { if (key == NULL || key->value == NULL) { errno = EINVAL; From b2bd353f62a5ebd7b9f879444f6bb2548b74982c Mon Sep 17 00:00:00 2001 From: Guy Benoish Date: Mon, 3 Feb 2020 17:19:00 +0530 Subject: [PATCH 0094/1280] Diskless-load emptyDb-related fixes 1. Call emptyDb even in case of diskless-load: We want modules to get the same FLUSHDB event as disk-based replication. 2. Do not fire any module events when flushing the backups array. 3. Delete redundant call to signalFlushedDb (Called from emptyDb). --- src/db.c | 57 ++++++++++++++++++++++++++++------------------- src/replication.c | 14 +++++++----- src/server.h | 1 + 3 files changed, 44 insertions(+), 28 deletions(-) diff --git a/src/db.c b/src/db.c index ba7be2725..c2f974cbb 100644 --- a/src/db.c +++ b/src/db.c @@ -347,7 +347,10 @@ robj *dbUnshareStringValue(redisDb *db, robj *key, robj *o) { * DB number if we want to flush only a single Redis database number. * * Flags are be EMPTYDB_NO_FLAGS if no special flags are specified or - * EMPTYDB_ASYNC if we want the memory to be freed in a different thread + * 1. EMPTYDB_ASYNC if we want the memory to be freed in a different thread. + * 2. EMPTYDB_BACKUP if we want to empty the backup dictionaries created by + * disklessLoadMakeBackups. In that case we only free memory and avoid + * firing module events. * and the function to return ASAP. * * On success the fuction returns the number of keys removed from the @@ -355,6 +358,8 @@ robj *dbUnshareStringValue(redisDb *db, robj *key, robj *o) { * DB number is out of range, and errno is set to EINVAL. */ long long emptyDbGeneric(redisDb *dbarray, int dbnum, int flags, void(callback)(void*)) { int async = (flags & EMPTYDB_ASYNC); + int backup = (flags & EMPTYDB_BACKUP); /* Just free the memory, nothing else */ + RedisModuleFlushInfoV1 fi = {REDISMODULE_FLUSHINFO_VERSION,!async,dbnum}; long long removed = 0; if (dbnum < -1 || dbnum >= server.dbnum) { @@ -362,16 +367,18 @@ long long emptyDbGeneric(redisDb *dbarray, int dbnum, int flags, void(callback)( return -1; } - /* Fire the flushdb modules event. */ - RedisModuleFlushInfoV1 fi = {REDISMODULE_FLUSHINFO_VERSION,!async,dbnum}; - moduleFireServerEvent(REDISMODULE_EVENT_FLUSHDB, - REDISMODULE_SUBEVENT_FLUSHDB_START, - &fi); + /* Pre-flush actions */ + if (!backup) { + /* Fire the flushdb modules event. */ + moduleFireServerEvent(REDISMODULE_EVENT_FLUSHDB, + REDISMODULE_SUBEVENT_FLUSHDB_START, + &fi); - /* Make sure the WATCHed keys are affected by the FLUSH* commands. - * Note that we need to call the function while the keys are still - * there. */ - signalFlushedDb(dbnum); + /* Make sure the WATCHed keys are affected by the FLUSH* commands. + * Note that we need to call the function while the keys are still + * there. */ + signalFlushedDb(dbnum); + } int startdb, enddb; if (dbnum == -1) { @@ -390,20 +397,24 @@ long long emptyDbGeneric(redisDb *dbarray, int dbnum, int flags, void(callback)( dictEmpty(dbarray[j].expires,callback); } } - if (server.cluster_enabled) { - if (async) { - slotToKeyFlushAsync(); - } else { - slotToKeyFlush(); - } - } - if (dbnum == -1) flushSlaveKeysWithExpireList(); - /* Also fire the end event. Note that this event will fire almost - * immediately after the start event if the flush is asynchronous. */ - moduleFireServerEvent(REDISMODULE_EVENT_FLUSHDB, - REDISMODULE_SUBEVENT_FLUSHDB_END, - &fi); + /* Post-flush actions */ + if (!backup) { + if (server.cluster_enabled) { + if (async) { + slotToKeyFlushAsync(); + } else { + slotToKeyFlush(); + } + } + if (dbnum == -1) flushSlaveKeysWithExpireList(); + + /* Also fire the end event. Note that this event will fire almost + * immediately after the start event if the flush is asynchronous. */ + moduleFireServerEvent(REDISMODULE_EVENT_FLUSHDB, + REDISMODULE_SUBEVENT_FLUSHDB_END, + &fi); + } return removed; } diff --git a/src/replication.c b/src/replication.c index b7e77184a..dd2c3dd2f 100644 --- a/src/replication.c +++ b/src/replication.c @@ -1339,8 +1339,8 @@ void disklessLoadRestoreBackups(redisDb *backup, int restore, int empty_db_flags server.db[i] = backup[i]; } } else { - /* Delete. */ - emptyDbGeneric(backup,-1,empty_db_flags,replicationEmptyDbCallback); + /* Delete (Pass EMPTYDB_BACKUP in order to avoid firing module events) . */ + emptyDbGeneric(backup,-1,empty_db_flags|EMPTYDB_BACKUP,replicationEmptyDbCallback); for (int i=0; i Date: Thu, 6 Feb 2020 14:53:33 +0200 Subject: [PATCH 0095/1280] add no_auth to COMMAND INFO --- src/server.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/server.c b/src/server.c index f3b7c37c5..01c048c90 100644 --- a/src/server.c +++ b/src/server.c @@ -3764,6 +3764,7 @@ void addReplyCommand(client *c, struct redisCommand *cmd) { flagcount += addReplyCommandFlag(c,cmd,CMD_SKIP_SLOWLOG, "skip_slowlog"); flagcount += addReplyCommandFlag(c,cmd,CMD_ASKING, "asking"); flagcount += addReplyCommandFlag(c,cmd,CMD_FAST, "fast"); + flagcount += addReplyCommandFlag(c,cmd,CMD_NO_AUTH, "no_auth"); if ((cmd->getkeys_proc && !(cmd->flags & CMD_MODULE)) || cmd->flags & CMD_MODULE_GETKEYS) { From 6c41cd18076346c21593f9700c96850baae3ff8e Mon Sep 17 00:00:00 2001 From: Guy Benoish Date: Thu, 6 Feb 2020 18:36:21 +0530 Subject: [PATCH 0096/1280] Fix memory leak in test_ld_conv --- tests/modules/misc.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/modules/misc.c b/tests/modules/misc.c index 41bec06ed..1048d5065 100644 --- a/tests/modules/misc.c +++ b/tests/modules/misc.c @@ -74,6 +74,7 @@ int test_ld_conv(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { RedisModule_ReplyWithError(ctx, err); goto final; } + /* Make sure we can't convert a string that has \0 in it */ char buf[4] = "123"; buf[1] = '\0'; @@ -81,8 +82,11 @@ int test_ld_conv(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { long double ld3; if (RedisModule_StringToLongDouble(s3, &ld3) == REDISMODULE_OK) { RedisModule_ReplyWithError(ctx, "Invalid string successfully converted to long double"); + RedisModule_FreeString(ctx, s3); goto final; } + RedisModule_FreeString(ctx, s3); + RedisModule_ReplyWithLongDouble(ctx, ld2); final: RedisModule_FreeString(ctx, s1); From 0cacccddc3c342815bbed8b123df94730cbda68f Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Thu, 6 Feb 2020 15:06:33 +0200 Subject: [PATCH 0097/1280] add no-slowlog option to RM_CreateCommand --- src/module.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/module.c b/src/module.c index 914c50df3..84a4527eb 100644 --- a/src/module.c +++ b/src/module.c @@ -730,6 +730,7 @@ int commandFlagsFromString(char *s) { else if (!strcasecmp(t,"random")) flags |= CMD_RANDOM; else if (!strcasecmp(t,"allow-stale")) flags |= CMD_STALE; else if (!strcasecmp(t,"no-monitor")) flags |= CMD_SKIP_MONITOR; + else if (!strcasecmp(t,"no-slowlog")) flags |= CMD_SKIP_SLOWLOG; else if (!strcasecmp(t,"fast")) flags |= CMD_FAST; else if (!strcasecmp(t,"no-auth")) flags |= CMD_NO_AUTH; else if (!strcasecmp(t,"getkeys-api")) flags |= CMD_MODULE_GETKEYS; @@ -781,6 +782,8 @@ int commandFlagsFromString(char *s) { * this means. * * **"no-monitor"**: Don't propagate the command on monitor. Use this if * the command has sensible data among the arguments. + * * **"no-slowlog"**: Don't log this command in the slowlog. Use this if + * the command has sensible data among the arguments. * * **"fast"**: The command time complexity is not greater * than O(log(N)) where N is the size of the collection or * anything else representing the normal scalability From bcc9361ad4df6b7f7483e99da08cd58484a7aef4 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 7 Feb 2020 14:03:43 +0100 Subject: [PATCH 0098/1280] Tracking: first conversion from hashing to key names. --- src/server.c | 4 +- src/server.h | 2 +- src/tracking.c | 206 +++++++++++++++++++++---------------------------- 3 files changed, 91 insertions(+), 121 deletions(-) diff --git a/src/server.c b/src/server.c index f87dfba46..df6a8c1f4 100644 --- a/src/server.c +++ b/src/server.c @@ -4221,7 +4221,7 @@ sds genRedisInfoString(const char *section) { "active_defrag_misses:%lld\r\n" "active_defrag_key_hits:%lld\r\n" "active_defrag_key_misses:%lld\r\n" - "tracking_used_slots:%lld\r\n", + "tracking_tracked_keys:%lld\r\n", server.stat_numconnections, server.stat_numcommands, getInstantaneousMetric(STATS_METRIC_COMMAND), @@ -4249,7 +4249,7 @@ sds genRedisInfoString(const char *section) { server.stat_active_defrag_misses, server.stat_active_defrag_key_hits, server.stat_active_defrag_key_misses, - trackingGetUsedSlots()); + (unsigned long long) trackingGetTotalItems()); } /* Replication */ diff --git a/src/server.h b/src/server.h index dfbc15ac3..a30db6b57 100644 --- a/src/server.h +++ b/src/server.h @@ -1654,7 +1654,7 @@ void trackingRememberKeys(client *c); void trackingInvalidateKey(robj *keyobj); void trackingInvalidateKeysOnFlush(int dbid); void trackingLimitUsedSlots(void); -unsigned long long trackingGetUsedSlots(void); +uint64_t trackingGetTotalItems(void); /* List data type */ void listTypeTryConversion(robj *subject, robj *value); diff --git a/src/tracking.c b/src/tracking.c index acb97800a..ecb7fbdcc 100644 --- a/src/tracking.c +++ b/src/tracking.c @@ -30,37 +30,22 @@ #include "server.h" -/* The tracking table is constituted by 2^24 radix trees (each tree, and the - * table itself, are allocated in a lazy way only when needed) tracking - * clients that may have certain keys in their local, client side, cache. - * - * Keys are grouped into 2^24 slots, in a way similar to Redis Cluster hash - * slots, however here the function we use is crc64, taking the least - * significant 24 bits of the output. +/* The tracking table is constituted by a radix tree of keys, each pointing + * to a radix tree of client IDs, used to track the clients that may have + * certain keys in their local, client side, cache. * * When a client enables tracking with "CLIENT TRACKING on", each key served to - * the client is hashed to one of such slots, and Redis will remember what - * client may have keys about such slot. Later, when a key in a given slot is - * modified, all the clients that may have local copies of keys in that slot - * will receive an invalidation message. There is no distinction of database - * number: a single table is used. + * the client is remembered in the table mapping the keys to the client IDs. + * Later, when a key is modified, all the clients that may have local copy + * of such key will receive an invalidation message. * * Clients will normally take frequently requested objects in memory, removing - * them when invalidation messages are received. A strategy clients may use is - * to just cache objects in a dictionary, associating to each cached object - * some incremental epoch, or just a timestamp. When invalidation messages are - * received clients may store, in a different table, the timestamp (or epoch) - * of the invalidation of such given slot: later when accessing objects, the - * eviction of stale objects may be performed in a lazy way by checking if the - * cached object timestamp is older than the invalidation timestamp for such - * objects. - * - * The output of the 24 bit hash function is very large (more than 16 million - * possible slots), so clients that may want to use less resources may only - * use the most significant bits instead of the full 24 bits. */ -#define TRACKING_TABLE_SIZE (1<<24) -rax **TrackingTable = NULL; -unsigned long TrackingTableUsedSlots = 0; + * them when invalidation messages are received. */ +rax *TrackingTable = NULL; +uint64_t TrackingTableTotalItems = 0; /* Total number of IDs stored across + the whole tracking table. This givesn + an hint about the total memory we + are using server side for CSC. */ robj *TrackingChannelName; /* Remove the tracking state from the client 'c'. Note that there is not much @@ -90,7 +75,7 @@ void enableTracking(client *c, uint64_t redirect_to) { c->client_tracking_redirection = redirect_to; server.tracking_clients++; if (TrackingTable == NULL) { - TrackingTable = zcalloc(sizeof(rax*) * TRACKING_TABLE_SIZE); + TrackingTable = raxNew(); TrackingChannelName = createStringObject("__redis__:invalidate",20); } } @@ -108,19 +93,20 @@ void trackingRememberKeys(client *c) { for(int j = 0; j < numkeys; j++) { int idx = keys[j]; sds sdskey = c->argv[idx]->ptr; - uint64_t hash = crc64(0, - (unsigned char*)sdskey,sdslen(sdskey))&(TRACKING_TABLE_SIZE-1); - if (TrackingTable[hash] == NULL) { - TrackingTable[hash] = raxNew(); - TrackingTableUsedSlots++; + rax *ids = raxFind(TrackingTable,(unsigned char*)sdskey,sdslen(sdskey)); + if (ids == raxNotFound) { + ids = raxNew(); + int inserted = raxTryInsert(TrackingTable,(unsigned char*)sdskey, + sdslen(sdskey),ids, NULL); + serverAssert(inserted == 1); } - raxTryInsert(TrackingTable[hash], - (unsigned char*)&c->id,sizeof(c->id),NULL,NULL); + if (raxTryInsert(ids,(unsigned char*)&c->id,sizeof(c->id),NULL,NULL)) + TrackingTableTotalItems++; } getKeysFreeResult(keys); } -void sendTrackingMessage(client *c, long long hash) { +void sendTrackingMessage(client *c, char *keyname, size_t keylen) { int using_redirection = 0; if (c->client_tracking_redirection) { client *redir = lookupClientByID(c->client_tracking_redirection); @@ -146,49 +132,44 @@ void sendTrackingMessage(client *c, long long hash) { if (c->resp > 2) { addReplyPushLen(c,2); addReplyBulkCBuffer(c,"invalidate",10); - addReplyLongLong(c,hash); + addReplyBulkCBuffer(c,keyname,keylen); } else if (using_redirection && c->flags & CLIENT_PUBSUB) { - robj *msg = createStringObjectFromLongLong(hash); - addReplyPubsubMessage(c,TrackingChannelName,msg); - decrRefCount(msg); + /* We use a static object to speedup things, however we assume + * that addReplyPubsubMessage() will not take a reference. */ + robj keyobj; + initStaticStringObject(keyobj,keyname); + addReplyPubsubMessage(c,TrackingChannelName,&keyobj); + serverAssert(keyobj.refcount == 1); } } -/* Invalidates a caching slot: this is actually the low level implementation - * of the API that Redis calls externally, that is trackingInvalidateKey(). */ -void trackingInvalidateSlot(uint64_t slot) { - if (TrackingTable == NULL || TrackingTable[slot] == NULL) return; - - raxIterator ri; - raxStart(&ri,TrackingTable[slot]); - raxSeek(&ri,"^",NULL,0); - while(raxNext(&ri)) { - uint64_t id; - memcpy(&id,ri.key,sizeof(id)); - client *c = lookupClientByID(id); - if (c == NULL || !(c->flags & CLIENT_TRACKING)) continue; - sendTrackingMessage(c,slot); - } - raxStop(&ri); - - /* Free the tracking table: we'll create the radix tree and populate it - * again if more keys will be modified in this caching slot. */ - raxFree(TrackingTable[slot]); - TrackingTable[slot] = NULL; - TrackingTableUsedSlots--; -} - /* This function is called from signalModifiedKey() or other places in Redis * when a key changes value. In the context of keys tracking, our task here is * to send a notification to every client that may have keys about such caching * slot. */ void trackingInvalidateKey(robj *keyobj) { - if (TrackingTable == NULL || TrackingTableUsedSlots == 0) return; - + if (TrackingTable == NULL) return; sds sdskey = keyobj->ptr; - uint64_t hash = crc64(0, - (unsigned char*)sdskey,sdslen(sdskey))&(TRACKING_TABLE_SIZE-1); - trackingInvalidateSlot(hash); + rax *ids = raxFind(TrackingTable,(unsigned char*)sdskey,sdslen(sdskey)); + if (ids == raxNotFound) return;; + + raxIterator ri; + raxStart(&ri,ids); + raxSeek(&ri,"^",NULL,0); + while(raxNext(&ri)) { + uint64_t id; + memcpy(&id,ri.key,sizeof(id)); + client *c = lookupClientByID(id); + if (c == NULL || !(c->flags & CLIENT_TRACKING)) continue; + sendTrackingMessage(c,sdskey,sdslen(sdskey)); + } + raxStop(&ri); + + /* Free the tracking table: we'll create the radix tree and populate it + * again if more keys will be modified in this caching slot. */ + TrackingTableTotalItems -= raxSize(ids); + raxFree(ids); + raxRemove(TrackingTable,(unsigned char*)sdskey,sdslen(sdskey),NULL); } /* This function is called when one or all the Redis databases are flushed @@ -205,6 +186,10 @@ void trackingInvalidateKey(robj *keyobj) { * we just send the invalidation message to all the clients, but don't * flush the table: it will slowly get garbage collected as more keys * are modified in the used caching slots. */ +void freeTrackingRadixTree(void *rt) { + raxFree(rt); +} + void trackingInvalidateKeysOnFlush(int dbid) { if (server.tracking_clients) { listNode *ln; @@ -213,84 +198,69 @@ void trackingInvalidateKeysOnFlush(int dbid) { while ((ln = listNext(&li)) != NULL) { client *c = listNodeValue(ln); if (c->flags & CLIENT_TRACKING) { - sendTrackingMessage(c,-1); + sendTrackingMessage(c,"",1); } } } /* In case of FLUSHALL, reclaim all the memory used by tracking. */ if (dbid == -1 && TrackingTable) { - for (int j = 0; j < TRACKING_TABLE_SIZE && TrackingTableUsedSlots > 0; j++) { - if (TrackingTable[j] != NULL) { - raxFree(TrackingTable[j]); - TrackingTable[j] = NULL; - TrackingTableUsedSlots--; - } - } - - /* If there are no clients with tracking enabled, we can even - * reclaim the memory used by the table itself. The code assumes - * the table is allocated only if there is at least one client alive - * with tracking enabled. */ - if (server.tracking_clients == 0) { - zfree(TrackingTable); - TrackingTable = NULL; - } + raxFreeWithCallback(TrackingTable,freeTrackingRadixTree); + TrackingTableTotalItems = 0; } } /* Tracking forces Redis to remember information about which client may have - * keys about certian caching slots. In workloads where there are a lot of - * reads, but keys are hardly modified, the amount of information we have - * to remember server side could be a lot: for each 16 millions of caching - * slots we may end with a radix tree containing many entries. + * certain keys. In workloads where there are a lot of reads, but keys are + * hardly modified, the amount of information we have to remember server side + * could be a lot, with the number of keys being totally not bound. * - * So Redis allows the user to configure a maximum fill rate for the + * So Redis allows the user to configure a maximum number of keys for the * invalidation table. This function makes sure that we don't go over the * specified fill rate: if we are over, we can just evict informations about - * random caching slots, and send invalidation messages to clients like if - * the key was modified. */ + * a random key, and send invalidation messages to clients like if the key was + * modified. */ void trackingLimitUsedSlots(void) { static unsigned int timeout_counter = 0; - + if (TrackingTable == NULL) return; if (server.tracking_table_max_fill == 0) return; /* No limits set. */ - unsigned int max_slots = - (TRACKING_TABLE_SIZE/100) * server.tracking_table_max_fill; - if (TrackingTableUsedSlots <= max_slots) { + size_t max_keys = server.tracking_table_max_fill; + if (raxSize(TrackingTable) <= max_keys) { timeout_counter = 0; return; /* Limit not reached. */ } - /* We have to invalidate a few slots to reach the limit again. The effort + /* We have to invalidate a few keys to reach the limit again. The effort * we do here is proportional to the number of times we entered this * function and found that we are still over the limit. */ int effort = 100 * (timeout_counter+1); - /* Let's start at a random position, and perform linear probing, in order - * to improve cache locality. However once we are able to find an used - * slot, jump again randomly, in order to avoid creating big holes in the - * table (that will make this funciton use more resourced later). */ + /* We just remove one key after another by using a random walk. */ + raxIterator ri; + raxStart(&ri,TrackingTable); while(effort > 0) { - unsigned int idx = rand() % TRACKING_TABLE_SIZE; - do { - effort--; - idx = (idx+1) % TRACKING_TABLE_SIZE; - if (TrackingTable[idx] != NULL) { - trackingInvalidateSlot(idx); - if (TrackingTableUsedSlots <= max_slots) { - timeout_counter = 0; - return; /* Return ASAP: we are again under the limit. */ - } else { - break; /* Jump to next random position. */ - } - } - } while(effort > 0); + effort--; + raxSeek(&ri,"^",NULL,0); + raxRandomWalk(&ri,0); + rax *ids = ri.data; + TrackingTableTotalItems -= raxSize(ids); + raxFree(ids); + raxRemove(TrackingTable,ri.key,ri.key_len,NULL); + if (raxSize(TrackingTable) <= max_keys) { + timeout_counter = 0; + raxStop(&ri); + return; /* Return ASAP: we are again under the limit. */ + } } + + /* If we reach this point, we were not able to go under the configured + * limit using the maximum effort we had for this run. */ + raxStop(&ri); timeout_counter++; } /* This is just used in order to access the amount of used slots in the * tracking table. */ -unsigned long long trackingGetUsedSlots(void) { - return TrackingTableUsedSlots; +uint64_t trackingGetTotalItems(void) { + return TrackingTableTotalItems; } From ddc2a9f5ee7a3ce9b2e3550aeb31cd9b763c852b Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 7 Feb 2020 17:19:11 +0100 Subject: [PATCH 0099/1280] Tracking: rename INFO field with total items. --- src/server.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/server.c b/src/server.c index df6a8c1f4..2cf8dbf19 100644 --- a/src/server.c +++ b/src/server.c @@ -4221,7 +4221,7 @@ sds genRedisInfoString(const char *section) { "active_defrag_misses:%lld\r\n" "active_defrag_key_hits:%lld\r\n" "active_defrag_key_misses:%lld\r\n" - "tracking_tracked_keys:%lld\r\n", + "tracking_total_items:%lld\r\n", server.stat_numconnections, server.stat_numcommands, getInstantaneousMetric(STATS_METRIC_COMMAND), From d5286419fc0c59775c031064270999d1d54505ea Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 7 Feb 2020 18:12:10 +0100 Subject: [PATCH 0100/1280] Rax.c: populate data field after random walk. --- src/rax.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/rax.c b/src/rax.c index 29b74ae90..a560dde02 100644 --- a/src/rax.c +++ b/src/rax.c @@ -1766,6 +1766,7 @@ int raxRandomWalk(raxIterator *it, size_t steps) { if (n->iskey) steps--; } it->node = n; + it->data = raxGetData(it->node); return 1; } From dcb43f4530c77c06f6ab7184cd00453f4824672e Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 7 Feb 2020 18:12:45 +0100 Subject: [PATCH 0101/1280] Tracking: minor change of names and new INFO field. --- src/config.c | 2 +- src/server.c | 2 ++ src/server.h | 3 ++- src/tracking.c | 8 ++++++-- 4 files changed, 11 insertions(+), 4 deletions(-) diff --git a/src/config.c b/src/config.c index a6b374817..b2e5fc12e 100644 --- a/src/config.c +++ b/src/config.c @@ -2160,7 +2160,7 @@ standardConfig configs[] = { createIntConfig("list-compress-depth", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.list_compress_depth, 0, INTEGER_CONFIG, NULL, NULL), createIntConfig("rdb-key-save-delay", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.rdb_key_save_delay, 0, INTEGER_CONFIG, NULL, NULL), createIntConfig("key-load-delay", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.key_load_delay, 0, INTEGER_CONFIG, NULL, NULL), - createIntConfig("tracking-table-max-fill", NULL, MODIFIABLE_CONFIG, 0, 100, server.tracking_table_max_fill, 10, INTEGER_CONFIG, NULL, NULL), /* Default: 10% tracking table max fill. */ + createIntConfig("tracking-table-max-fill", NULL, MODIFIABLE_CONFIG, 0, 100, server.tracking_table_max_keys, 1000000, INTEGER_CONFIG, NULL, NULL), /* Default: 10% tracking table max number of keys tracked. */ createIntConfig("active-expire-effort", NULL, MODIFIABLE_CONFIG, 1, 10, server.active_expire_effort, 1, INTEGER_CONFIG, NULL, NULL), /* From 1 to 10. */ createIntConfig("hz", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.config_hz, CONFIG_DEFAULT_HZ, INTEGER_CONFIG, NULL, updateHZ), createIntConfig("min-replicas-to-write", "min-slaves-to-write", MODIFIABLE_CONFIG, 0, INT_MAX, server.repl_min_slaves_to_write, 0, INTEGER_CONFIG, NULL, updateGoodSlaves), diff --git a/src/server.c b/src/server.c index 2cf8dbf19..910cf5410 100644 --- a/src/server.c +++ b/src/server.c @@ -4221,6 +4221,7 @@ sds genRedisInfoString(const char *section) { "active_defrag_misses:%lld\r\n" "active_defrag_key_hits:%lld\r\n" "active_defrag_key_misses:%lld\r\n" + "tracking_total_keys:%lld\r\n" "tracking_total_items:%lld\r\n", server.stat_numconnections, server.stat_numcommands, @@ -4249,6 +4250,7 @@ sds genRedisInfoString(const char *section) { server.stat_active_defrag_misses, server.stat_active_defrag_key_hits, server.stat_active_defrag_key_misses, + (unsigned long long) trackingGetTotalKeys(), (unsigned long long) trackingGetTotalItems()); } diff --git a/src/server.h b/src/server.h index a30db6b57..3e055a7db 100644 --- a/src/server.h +++ b/src/server.h @@ -1306,7 +1306,7 @@ struct redisServer { list *ready_keys; /* List of readyList structures for BLPOP & co */ /* Client side caching. */ unsigned int tracking_clients; /* # of clients with tracking enabled.*/ - int tracking_table_max_fill; /* Max fill percentage. */ + int tracking_table_max_keys; /* Max number of keys in tracking table. */ /* Sort parameters - qsort_r() is only available under BSD so we * have to take this state global, in order to pass it to sortCompare() */ int sort_desc; @@ -1655,6 +1655,7 @@ void trackingInvalidateKey(robj *keyobj); void trackingInvalidateKeysOnFlush(int dbid); void trackingLimitUsedSlots(void); uint64_t trackingGetTotalItems(void); +uint64_t trackingGetTotalKeys(void); /* List data type */ void listTypeTryConversion(robj *subject, robj *value); diff --git a/src/tracking.c b/src/tracking.c index ecb7fbdcc..9c1c9620c 100644 --- a/src/tracking.c +++ b/src/tracking.c @@ -223,8 +223,8 @@ void trackingInvalidateKeysOnFlush(int dbid) { void trackingLimitUsedSlots(void) { static unsigned int timeout_counter = 0; if (TrackingTable == NULL) return; - if (server.tracking_table_max_fill == 0) return; /* No limits set. */ - size_t max_keys = server.tracking_table_max_fill; + if (server.tracking_table_max_keys == 0) return; /* No limits set. */ + size_t max_keys = server.tracking_table_max_keys; if (raxSize(TrackingTable) <= max_keys) { timeout_counter = 0; return; /* Limit not reached. */ @@ -264,3 +264,7 @@ void trackingLimitUsedSlots(void) { uint64_t trackingGetTotalItems(void) { return TrackingTableTotalItems; } + +uint64_t trackingGetTotalKeys(void) { + return raxSize(TrackingTable); +} From dda3f1636ae371c28a1a2003499e36d73f309459 Mon Sep 17 00:00:00 2001 From: Seunghoon Woo Date: Mon, 10 Feb 2020 16:32:46 +0900 Subject: [PATCH 0102/1280] [FIX] revisit CVE-2015-8080 vulnerability --- deps/lua/src/lua_struct.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/deps/lua/src/lua_struct.c b/deps/lua/src/lua_struct.c index 4d5f027b8..c58c8e72b 100644 --- a/deps/lua/src/lua_struct.c +++ b/deps/lua/src/lua_struct.c @@ -89,12 +89,14 @@ typedef struct Header { } Header; -static int getnum (const char **fmt, int df) { +static int getnum (lua_State *L, const char **fmt, int df) { if (!isdigit(**fmt)) /* no number? */ return df; /* return default value */ else { int a = 0; do { + if (a > (INT_MAX / 10) || a * 10 > (INT_MAX - (**fmt - '0'))) + luaL_error(L, "integral size overflow"); a = a*10 + *((*fmt)++) - '0'; } while (isdigit(**fmt)); return a; @@ -115,9 +117,9 @@ static size_t optsize (lua_State *L, char opt, const char **fmt) { case 'f': return sizeof(float); case 'd': return sizeof(double); case 'x': return 1; - case 'c': return getnum(fmt, 1); + case 'c': return getnum(L, fmt, 1); case 'i': case 'I': { - int sz = getnum(fmt, sizeof(int)); + int sz = getnum(L, fmt, sizeof(int)); if (sz > MAXINTSIZE) luaL_error(L, "integral size %d is larger than limit of %d", sz, MAXINTSIZE); @@ -150,7 +152,7 @@ static void controloptions (lua_State *L, int opt, const char **fmt, case '>': h->endian = BIG; return; case '<': h->endian = LITTLE; return; case '!': { - int a = getnum(fmt, MAXALIGN); + int a = getnum(L, fmt, MAXALIGN); if (!isp2(a)) luaL_error(L, "alignment %d is not a power of 2", a); h->align = a; From 054932343e99cc5ebd06513a1557f25b442d71ed Mon Sep 17 00:00:00 2001 From: "meir@redislabs.com" Date: Mon, 10 Feb 2020 12:10:32 +0200 Subject: [PATCH 0103/1280] Changed log level for module fork api from 'notice' to 'verbos'. --- src/module.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/module.c b/src/module.c index ed79ee226..d15aa6c93 100644 --- a/src/module.c +++ b/src/module.c @@ -6706,7 +6706,7 @@ int RM_Fork(RedisModuleForkDoneHandler cb, void *user_data) { server.module_child_pid = childpid; moduleForkInfo.done_handler = cb; moduleForkInfo.done_handler_user_data = user_data; - serverLog(LL_NOTICE, "Module fork started pid: %d ", childpid); + serverLog(LL_VERBOSE, "Module fork started pid: %d ", childpid); } return childpid; } @@ -6729,7 +6729,7 @@ int TerminateModuleForkChild(int child_pid, int wait) { server.module_child_pid != child_pid) return C_ERR; int statloc; - serverLog(LL_NOTICE,"Killing running module fork child: %ld", + serverLog(LL_VERBOSE,"Killing running module fork child: %ld", (long) server.module_child_pid); if (kill(server.module_child_pid,SIGUSR1) != -1 && wait) { while(wait4(server.module_child_pid,&statloc,0,NULL) != From f7dccdf3d089e668e2c4fa494e90bb91cc204f08 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 10 Feb 2020 13:42:18 +0100 Subject: [PATCH 0104/1280] Tracking: always reply with an array of keys. --- src/pubsub.c | 8 ++++++-- src/tracking.c | 5 ++++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/pubsub.c b/src/pubsub.c index 994dd9734..5cb4298e0 100644 --- a/src/pubsub.c +++ b/src/pubsub.c @@ -35,7 +35,11 @@ int clientSubscriptionsCount(client *c); * Pubsub client replies API *----------------------------------------------------------------------------*/ -/* Send a pubsub message of type "message" to the client. */ +/* Send a pubsub message of type "message" to the client. + * Normally 'msg' is a Redis object containing the string to send as + * message. However if the caller sets 'msg' as NULL, it will be able + * to send a special message (for instance an Array type) by using the + * addReply*() API family. */ void addReplyPubsubMessage(client *c, robj *channel, robj *msg) { if (c->resp == 2) addReply(c,shared.mbulkhdr[3]); @@ -43,7 +47,7 @@ void addReplyPubsubMessage(client *c, robj *channel, robj *msg) { addReplyPushLen(c,3); addReply(c,shared.messagebulk); addReplyBulk(c,channel); - addReplyBulk(c,msg); + if (msg) addReplyBulk(c,msg); } /* Send a pubsub message of type "pmessage" to the client. The difference diff --git a/src/tracking.c b/src/tracking.c index 9c1c9620c..3122563ac 100644 --- a/src/tracking.c +++ b/src/tracking.c @@ -132,13 +132,16 @@ void sendTrackingMessage(client *c, char *keyname, size_t keylen) { if (c->resp > 2) { addReplyPushLen(c,2); addReplyBulkCBuffer(c,"invalidate",10); + addReplyArrayLen(c,1); addReplyBulkCBuffer(c,keyname,keylen); } else if (using_redirection && c->flags & CLIENT_PUBSUB) { /* We use a static object to speedup things, however we assume * that addReplyPubsubMessage() will not take a reference. */ robj keyobj; initStaticStringObject(keyobj,keyname); - addReplyPubsubMessage(c,TrackingChannelName,&keyobj); + addReplyPubsubMessage(c,TrackingChannelName,NULL); + addReplyArrayLen(c,1); + addReplyBulk(c,&keyobj); serverAssert(keyobj.refcount == 1); } } From c7973267422c976cb5db87c8d1f68e1bbcbb2877 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 10 Feb 2020 17:18:11 +0100 Subject: [PATCH 0105/1280] Tracking: BCAST: parsing of the options + skeleton. --- src/networking.c | 63 +++++++++++++++++++++++++++++++++++++----------- src/server.c | 5 +++- src/server.h | 8 +++++- src/tracking.c | 16 +++++++++--- 4 files changed, 73 insertions(+), 19 deletions(-) diff --git a/src/networking.c b/src/networking.c index 2b0f7464a..690a134f1 100644 --- a/src/networking.c +++ b/src/networking.c @@ -154,6 +154,7 @@ client *createClient(connection *conn) { c->peerid = NULL; c->client_list_node = NULL; c->client_tracking_redirection = 0; + c->client_tracking_prefix_nodes = NULL; c->auth_callback = NULL; c->auth_callback_privdata = NULL; c->auth_module = NULL; @@ -2219,38 +2220,72 @@ NULL UNIT_MILLISECONDS) != C_OK) return; pauseClients(duration); addReply(c,shared.ok); - } else if (!strcasecmp(c->argv[1]->ptr,"tracking") && - (c->argc == 3 || c->argc == 5)) - { - /* CLIENT TRACKING (on|off) [REDIRECT ] */ + } else if (!strcasecmp(c->argv[1]->ptr,"tracking") && c->argc >= 3) { + /* CLIENT TRACKING (on|off) [REDIRECT ] [BCAST] [PREFIX first] + * [PREFIX second] ... */ long long redir = 0; + int bcast = 0; + robj **prefix; + size_t numprefix = 0; - /* Parse the redirection option: we'll require the client with - * the specified ID to exist right now, even if it is possible - * it will get disconnected later. */ - if (c->argc == 5) { - if (strcasecmp(c->argv[3]->ptr,"redirect") != 0) { - addReply(c,shared.syntaxerr); - return; - } else { - if (getLongLongFromObjectOrReply(c,c->argv[4],&redir,NULL) != + /* Parse the options. */ + if (for int j = 3; j < argc; j++) { + int moreargs = (c->argc-1) - j; + + if (!strcasecmp(c->argv[j]->ptr,"redirect") && moreargs) { + j++; + if (getLongLongFromObjectOrReply(c,c->argv[j],&redir,NULL) != C_OK) return; + /* We will require the client with the specified ID to exist + * right now, even if it is possible that it gets disconnected + * later. Still a valid sanity check. */ if (lookupClientByID(redir) == NULL) { addReplyError(c,"The client ID you want redirect to " "does not exist"); return; } + } else if (!strcasecmp(c->argv[j]->ptr,"bcast")) { + bcast++; + } else if (!strcasecmp(c->argv[j]->ptr,"prefix") && morearg) { + j++; + prefix = zrealloc(sizeof(robj*)*(numprefix+1)); + prefix[numprefix++] = argv[j]; + } else { + addReply(c,shared.syntaxerr); + return; } } + /* Make sure options are compatible among each other and with the + * current state of the client. */ + if (!bcast && numprefix) { + addReplyError("PREFIX option requires BCAST mode to be enabled"); + zfree(prefix); + return; + } + + if (client->flags & CLIENT_TRACKING) { + int oldbcast = !!client->flags & CLIENT_TRACKING_BCAST; + if (oldbcast != bcast) { + } + addReplyError( + "You can't switch BCAST mode on/off before disabling " + "tracking for this client, and then re-enabling it with " + "a different mode."); + zfree(prefix); + return; + } + + /* Options are ok: enable or disable the tracking for this client. */ if (!strcasecmp(c->argv[2]->ptr,"on")) { - enableTracking(c,redir); + enableTracking(c,redir,bcast,prefix,numprefix); } else if (!strcasecmp(c->argv[2]->ptr,"off")) { disableTracking(c); } else { addReply(c,shared.syntaxerr); return; } + zfree(prefix); addReply(c,shared.ok); } else if (!strcasecmp(c->argv[1]->ptr,"getredir") && c->argc == 2) { /* CLIENT GETREDIR */ diff --git a/src/server.c b/src/server.c index 910cf5410..1001fa4f7 100644 --- a/src/server.c +++ b/src/server.c @@ -3310,8 +3310,11 @@ void call(client *c, int flags) { if (c->cmd->flags & CMD_READONLY) { client *caller = (c->flags & CLIENT_LUA && server.lua_caller) ? server.lua_caller : c; - if (caller->flags & CLIENT_TRACKING) + if (caller->flags & CLIENT_TRACKING && + !(caller->flags & CLIENT_TRACKING_BCAST)) + { trackingRememberKeys(caller); + } } server.fixed_time_expire--; diff --git a/src/server.h b/src/server.h index 3e055a7db..d3ca0d01b 100644 --- a/src/server.h +++ b/src/server.h @@ -247,6 +247,7 @@ typedef long long ustime_t; /* microsecond time type. */ #define CLIENT_TRACKING (1ULL<<31) /* Client enabled keys tracking in order to perform client side caching. */ #define CLIENT_TRACKING_BROKEN_REDIR (1ULL<<32) /* Target client is invalid. */ +#define CLIENT_TRACKING_BCAST (1ULL<<33) /* Tracking in BCAST mode. */ /* Client block type (btype field in client structure) * if CLIENT_BLOCKED flag is set. */ @@ -822,6 +823,11 @@ typedef struct client { * invalidation messages for keys fetched by this client will be send to * the specified client ID. */ uint64_t client_tracking_redirection; + list *client_tracking_prefix_nodes; /* This list contains listNode pointers + to the nodes we have in every list + of clients in the tracking bcast + table. This way we can remove our + client in O(1) for each list. */ /* Response buffer */ int bufpos; @@ -1648,7 +1654,7 @@ void addReplyStatusFormat(client *c, const char *fmt, ...); #endif /* Client side caching (tracking mode) */ -void enableTracking(client *c, uint64_t redirect_to); +void enableTracking(client *c, uint64_t redirect_to, int bcast, robj **prefix, size_t numprefix); void disableTracking(client *c); void trackingRememberKeys(client *c); void trackingInvalidateKey(robj *keyobj); diff --git a/src/tracking.c b/src/tracking.c index 3122563ac..413b21328 100644 --- a/src/tracking.c +++ b/src/tracking.c @@ -42,6 +42,7 @@ * Clients will normally take frequently requested objects in memory, removing * them when invalidation messages are received. */ rax *TrackingTable = NULL; +rax *PrefixTable = NULL; uint64_t TrackingTableTotalItems = 0; /* Total number of IDs stored across the whole tracking table. This givesn an hint about the total memory we @@ -68,16 +69,25 @@ void disableTracking(client *c) { * eventually get freed, we'll send a message to the original client to * inform it of the condition. Multiple clients can redirect the invalidation * messages to the same client ID. */ -void enableTracking(client *c, uint64_t redirect_to) { - if (c->flags & CLIENT_TRACKING) return; +void enableTracking(client *c, uint64_t redirect_to, int bcast, robj **prefix, size_t numprefix) { c->flags |= CLIENT_TRACKING; c->flags &= ~CLIENT_TRACKING_BROKEN_REDIR; c->client_tracking_redirection = redirect_to; - server.tracking_clients++; + if (!(c->flags & CLIENT_TRACKING)) server.tracking_clients++; if (TrackingTable == NULL) { TrackingTable = raxNew(); + PrefixTable = raxNew(); TrackingChannelName = createStringObject("__redis__:invalidate",20); } + + if (bcast) { + c->flags |= CLIENT_TRACKING_BCAST; + if (numprefix == 0) enableBcastTrackingForPrefix(c,"",0); + for (int j = 0; j < numprefix; j++) { + sds sdsprefix = prefix[j]->ptr; + enableBcastTrackingForPrefix(c,sdsprefix,sdslen(prefix)); + } + } } /* This function is called after the excution of a readonly command in the From 6cd5fa3338bed65bdeb66189481c1a0ae7eee377 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 11 Feb 2020 17:26:27 +0100 Subject: [PATCH 0106/1280] Tracking: BCAST: registration in the prefix table. --- src/networking.c | 21 +++++++++--------- src/server.h | 9 +++----- src/tracking.c | 57 +++++++++++++++++++++++++++++++++++++++++++++--- 3 files changed, 67 insertions(+), 20 deletions(-) diff --git a/src/networking.c b/src/networking.c index 690a134f1..344b76260 100644 --- a/src/networking.c +++ b/src/networking.c @@ -154,7 +154,7 @@ client *createClient(connection *conn) { c->peerid = NULL; c->client_list_node = NULL; c->client_tracking_redirection = 0; - c->client_tracking_prefix_nodes = NULL; + c->client_tracking_prefixes = NULL; c->auth_callback = NULL; c->auth_callback_privdata = NULL; c->auth_module = NULL; @@ -2028,7 +2028,6 @@ int clientSetNameOrReply(client *c, robj *name) { void clientCommand(client *c) { listNode *ln; listIter li; - client *client; if (c->argc == 2 && !strcasecmp(c->argv[1]->ptr,"help")) { const char *help[] = { @@ -2142,7 +2141,7 @@ NULL /* Iterate clients killing all the matching clients. */ listRewind(server.clients,&li); while ((ln = listNext(&li)) != NULL) { - client = listNodeValue(ln); + client *client = listNodeValue(ln); if (addr && strcmp(getClientPeerId(client),addr) != 0) continue; if (type != -1 && getClientType(client) != type) continue; if (id != 0 && client->id != id) continue; @@ -2229,7 +2228,7 @@ NULL size_t numprefix = 0; /* Parse the options. */ - if (for int j = 3; j < argc; j++) { + for (int j = 3; j < c->argc; j++) { int moreargs = (c->argc-1) - j; if (!strcasecmp(c->argv[j]->ptr,"redirect") && moreargs) { @@ -2246,10 +2245,10 @@ NULL } } else if (!strcasecmp(c->argv[j]->ptr,"bcast")) { bcast++; - } else if (!strcasecmp(c->argv[j]->ptr,"prefix") && morearg) { + } else if (!strcasecmp(c->argv[j]->ptr,"prefix") && moreargs) { j++; - prefix = zrealloc(sizeof(robj*)*(numprefix+1)); - prefix[numprefix++] = argv[j]; + prefix = zrealloc(prefix,sizeof(robj*)*(numprefix+1)); + prefix[numprefix++] = c->argv[j]; } else { addReply(c,shared.syntaxerr); return; @@ -2259,16 +2258,16 @@ NULL /* Make sure options are compatible among each other and with the * current state of the client. */ if (!bcast && numprefix) { - addReplyError("PREFIX option requires BCAST mode to be enabled"); + addReplyError(c,"PREFIX option requires BCAST mode to be enabled"); zfree(prefix); return; } - if (client->flags & CLIENT_TRACKING) { - int oldbcast = !!client->flags & CLIENT_TRACKING_BCAST; + if (c->flags & CLIENT_TRACKING) { + int oldbcast = !!c->flags & CLIENT_TRACKING_BCAST; if (oldbcast != bcast) { } - addReplyError( + addReplyError(c, "You can't switch BCAST mode on/off before disabling " "tracking for this client, and then re-enabling it with " "a different mode."); diff --git a/src/server.h b/src/server.h index d3ca0d01b..725c3cbc8 100644 --- a/src/server.h +++ b/src/server.h @@ -823,12 +823,9 @@ typedef struct client { * invalidation messages for keys fetched by this client will be send to * the specified client ID. */ uint64_t client_tracking_redirection; - list *client_tracking_prefix_nodes; /* This list contains listNode pointers - to the nodes we have in every list - of clients in the tracking bcast - table. This way we can remove our - client in O(1) for each list. */ - + rax *client_tracking_prefixes; /* A dictionary of prefixes we are already + subscribed to in BCAST mode, in the + context of client side caching. */ /* Response buffer */ int bufpos; char buf[PROTO_REPLY_CHUNK_BYTES]; diff --git a/src/tracking.c b/src/tracking.c index 413b21328..9f46275a4 100644 --- a/src/tracking.c +++ b/src/tracking.c @@ -49,6 +49,15 @@ uint64_t TrackingTableTotalItems = 0; /* Total number of IDs stored across are using server side for CSC. */ robj *TrackingChannelName; +/* This is the structure that we have as value of the PrefixTable, and + * represents the list of keys modified, and the list of clients that need + * to be notified, for a given prefix. */ +typedef struct bcastState { + rax *keys; /* Keys modified in the current event loop cycle. */ + rax *clients; /* Clients subscribed to the notification events for this + prefix. */ +} bcastState; + /* Remove the tracking state from the client 'c'. Note that there is not much * to do for us here, if not to decrement the counter of the clients in * tracking mode, because we just store the ID of the client in the tracking @@ -56,9 +65,51 @@ robj *TrackingChannelName; * client with many entries in the table is removed, it would cost a lot of * time to do the cleanup. */ void disableTracking(client *c) { + /* If this client is in broadcasting mode, we need to unsubscribe it + * from all the prefixes it is registered to. */ + if (c->flags & CLIENT_TRACKING_BCAST) { + raxIterator ri; + raxStart(&ri,c->client_tracking_prefixes); + raxSeek(&ri,"^",NULL,0); + while(raxNext(&ri)) { + bcastState *bs = raxFind(PrefixTable,ri.key,ri.key_len); + serverAssert(bs != raxNotFound); + raxRemove(bs->clients,(unsigned char*)&c,sizeof(c),NULL); + /* Was it the last client? Remove the prefix from the + * table. */ + if (raxSize(bs->clients) == 0) { + raxFree(bs->clients); + raxFree(bs->keys); + zfree(bs); + raxRemove(PrefixTable,ri.key,ri.key_len,NULL); + } + } + raxStop(&ri); + } + + /* Clear flags and adjust the count. */ if (c->flags & CLIENT_TRACKING) { server.tracking_clients--; - c->flags &= ~(CLIENT_TRACKING|CLIENT_TRACKING_BROKEN_REDIR); + c->flags &= ~(CLIENT_TRACKING|CLIENT_TRACKING_BROKEN_REDIR| + CLIENT_TRACKING_BCAST); + } +} + +/* Set the client 'c' to track the prefix 'prefix'. If the client 'c' is + * already registered for the specified prefix, no operation is performed. */ +void enableBcastTrackingForPrefix(client *c, char *prefix, size_t plen) { + bcastState *bs = raxFind(PrefixTable,(unsigned char*)prefix,sdslen(prefix)); + /* If this is the first client subscribing to such prefix, create + * the prefix in the table. */ + if (bs == raxNotFound) { + bs = zmalloc(sizeof(*bs)); + bs->keys = raxNew(); + bs->clients = raxNew(); + raxInsert(PrefixTable,(unsigned char*)prefix,plen,bs,NULL); + } + if (raxTryInsert(bs->clients,(unsigned char*)&c,sizeof(c),NULL,NULL)) { + raxInsert(c->client_tracking_prefixes, + (unsigned char*)prefix,plen,NULL,NULL); } } @@ -83,9 +134,9 @@ void enableTracking(client *c, uint64_t redirect_to, int bcast, robj **prefix, s if (bcast) { c->flags |= CLIENT_TRACKING_BCAST; if (numprefix == 0) enableBcastTrackingForPrefix(c,"",0); - for (int j = 0; j < numprefix; j++) { + for (size_t j = 0; j < numprefix; j++) { sds sdsprefix = prefix[j]->ptr; - enableBcastTrackingForPrefix(c,sdsprefix,sdslen(prefix)); + enableBcastTrackingForPrefix(c,sdsprefix,sdslen(sdsprefix)); } } } From 07a92b811856a30daaf28dc326876b8ecbd1aeeb Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 11 Feb 2020 18:11:59 +0100 Subject: [PATCH 0107/1280] Tracking: BCAST: broadcasting of keys in prefixes implemented. --- src/server.h | 1 + src/tracking.c | 104 ++++++++++++++++++++++++++++++++++++++++++++----- 2 files changed, 95 insertions(+), 10 deletions(-) diff --git a/src/server.h b/src/server.h index 725c3cbc8..439bbc393 100644 --- a/src/server.h +++ b/src/server.h @@ -1659,6 +1659,7 @@ void trackingInvalidateKeysOnFlush(int dbid); void trackingLimitUsedSlots(void); uint64_t trackingGetTotalItems(void); uint64_t trackingGetTotalKeys(void); +void trackingBroadcastInvalidationMessages(void); /* List data type */ void listTypeTryConversion(robj *subject, robj *value); diff --git a/src/tracking.c b/src/tracking.c index 9f46275a4..345c5f1ad 100644 --- a/src/tracking.c +++ b/src/tracking.c @@ -167,7 +167,17 @@ void trackingRememberKeys(client *c) { getKeysFreeResult(keys); } -void sendTrackingMessage(client *c, char *keyname, size_t keylen) { +/* Given a key name, this function sends an invalidation message in the + * proper channel (depending on RESP version: PubSub or Push message) and + * to the proper client (in case fo redirection), in the context of the + * client 'c' with tracking enabled. + * + * In case the 'proto' argument is non zero, the function will assume that + * 'keyname' points to a buffer of 'keylen' bytes already expressed in the + * form of Redis RESP protocol, representing an array of keys to send + * to the client as value of the invalidation. This is used in BCAST mode + * in order to optimized the implementation to use less CPU time. */ +void sendTrackingMessage(client *c, char *keyname, size_t keylen, int proto) { int using_redirection = 0; if (c->client_tracking_redirection) { client *redir = lookupClientByID(c->client_tracking_redirection); @@ -193,18 +203,38 @@ void sendTrackingMessage(client *c, char *keyname, size_t keylen) { if (c->resp > 2) { addReplyPushLen(c,2); addReplyBulkCBuffer(c,"invalidate",10); - addReplyArrayLen(c,1); - addReplyBulkCBuffer(c,keyname,keylen); } else if (using_redirection && c->flags & CLIENT_PUBSUB) { /* We use a static object to speedup things, however we assume * that addReplyPubsubMessage() will not take a reference. */ - robj keyobj; - initStaticStringObject(keyobj,keyname); addReplyPubsubMessage(c,TrackingChannelName,NULL); - addReplyArrayLen(c,1); - addReplyBulk(c,&keyobj); - serverAssert(keyobj.refcount == 1); } + + /* Send the "value" part, which is the array of keys. */ + if (proto) { + addReplyProto(c,keyname,keylen); + } else { + addReplyArrayLen(c,1); + addReplyBulkCBuffer(c,keyname,keylen); + } +} + +/* This function is called when a key is modified in Redis and in the case + * we have at least one client with the BCAST mode enabled. + * Its goal is to set the key in the right broadcast state if the key + * matches one or more prefixes in the prefix table. Later when we + * return to the event loop, we'll send invalidation messages to the + * clients subscribed to each prefix. */ +void trackingRememberKeyToBroadcast(char *keyname, size_t keylen) { + raxIterator ri; + raxStart(&ri,PrefixTable); + raxSeek(&ri,"^",NULL,0); + while(raxNext(&ri)) { + if (keylen > ri.key_len) continue; + if (memcmp(ri.key,keyname,ri.key_len) != 0) continue; + bcastState *bs = ri.data; + raxTryInsert(bs->keys,(unsigned char*)keyname,keylen,NULL,NULL); + } + raxStop(&ri); } /* This function is called from signalModifiedKey() or other places in Redis @@ -214,6 +244,10 @@ void sendTrackingMessage(client *c, char *keyname, size_t keylen) { void trackingInvalidateKey(robj *keyobj) { if (TrackingTable == NULL) return; sds sdskey = keyobj->ptr; + + if (raxSize(PrefixTable) > 0) + trackingRememberKeyToBroadcast(sdskey,sdslen(sdskey)); + rax *ids = raxFind(TrackingTable,(unsigned char*)sdskey,sdslen(sdskey)); if (ids == raxNotFound) return;; @@ -225,7 +259,7 @@ void trackingInvalidateKey(robj *keyobj) { memcpy(&id,ri.key,sizeof(id)); client *c = lookupClientByID(id); if (c == NULL || !(c->flags & CLIENT_TRACKING)) continue; - sendTrackingMessage(c,sdskey,sdslen(sdskey)); + sendTrackingMessage(c,sdskey,sdslen(sdskey),0); } raxStop(&ri); @@ -262,7 +296,7 @@ void trackingInvalidateKeysOnFlush(int dbid) { while ((ln = listNext(&li)) != NULL) { client *c = listNodeValue(ln); if (c->flags & CLIENT_TRACKING) { - sendTrackingMessage(c,"",1); + sendTrackingMessage(c,"",1,0); } } } @@ -323,6 +357,56 @@ void trackingLimitUsedSlots(void) { timeout_counter++; } +/* This function will run the prefixes of clients in BCAST mode and + * keys that were modified about each prefix, and will send the + * notifications to each client in each prefix. */ +void trackingBroadcastInvalidationMessages(void) { + raxIterator ri, ri2; + raxStart(&ri,PrefixTable); + raxSeek(&ri,"^",NULL,0); + while(raxNext(&ri)) { + bcastState *bs = ri.data; + /* Create the array reply with the list of keys once, then send + * it to all the clients subscribed to this prefix. */ + char buf[32]; + size_t len = ll2string(buf,sizeof(buf),raxSize(bs->keys)); + sds proto = sdsempty(); + proto = sdsMakeRoomFor(proto,raxSize(bs->keys)*15); + proto = sdscatlen(proto,"*",1); + proto = sdscatlen(proto,buf,len); + proto = sdscatlen(proto,"\r\n",2); + raxStart(&ri2,bs->keys); + raxSeek(&ri2,"^",NULL,0); + while(raxNext(&ri2)) { + len = ll2string(buf,sizeof(buf),ri2.key_len); + sds proto = sdsnewlen("$",1); + proto = sdscatlen(proto,ri2.key,ri2.key_len); + proto = sdscatlen(proto,"\r\n",2); + } + raxStop(&ri2); + + /* Send this array of keys to every client in the list. */ + raxStart(&ri2,bs->clients); + raxSeek(&ri2,"^",NULL,0); + while(raxNext(&ri2)) { + client *c; + memcpy(&c,ri2.key,sizeof(c)); + sendTrackingMessage(c,proto,sdslen(proto),1); + } + raxStop(&ri2); + + /* Clean up: we can remove everything from this state, because we + * want to only track the new keys that will be accumulated starting + * from now. */ + sdsfree(proto); + raxFree(bs->clients); + raxFree(bs->keys); + bs->clients = raxNew(); + bs->keys = raxNew(); + } + raxStop(&ri); +} + /* This is just used in order to access the amount of used slots in the * tracking table. */ uint64_t trackingGetTotalItems(void) { From 9ae32c1420af3374fc61cecc869290afd1f1a958 Mon Sep 17 00:00:00 2001 From: lifubang Date: Wed, 12 Feb 2020 16:34:22 +0800 Subject: [PATCH 0108/1280] correct help info for --user and --pass Signed-off-by: lifubang --- src/redis-cli.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/redis-cli.c b/src/redis-cli.c index c1bda0a00..1d79e0db0 100644 --- a/src/redis-cli.c +++ b/src/redis-cli.c @@ -1688,8 +1688,8 @@ static void usage(void) { " You can also use the " REDIS_CLI_AUTH_ENV " environment\n" " variable to pass this password more safely\n" " (if both are used, this argument takes predecence).\n" -" -user Used to send ACL style 'AUTH username pass'. Needs -a.\n" -" -pass Alias of -a for consistency with the new --user option.\n" +" --user Used to send ACL style 'AUTH username pass'. Needs -a.\n" +" --pass Alias of -a for consistency with the new --user option.\n" " -u Server URI.\n" " -r Execute specified command N times.\n" " -i When -r is used, waits seconds per command.\n" From c98236a5f37ecdeb822492c7e514a2424fe33194 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 12 Feb 2020 19:22:04 +0100 Subject: [PATCH 0109/1280] Tracking: BCAST: basic feature now works. --- src/networking.c | 2 +- src/server.c | 4 +++ src/tracking.c | 89 +++++++++++++++++++++++++++--------------------- 3 files changed, 55 insertions(+), 40 deletions(-) diff --git a/src/networking.c b/src/networking.c index 344b76260..46534253e 100644 --- a/src/networking.c +++ b/src/networking.c @@ -2224,7 +2224,7 @@ NULL * [PREFIX second] ... */ long long redir = 0; int bcast = 0; - robj **prefix; + robj **prefix = NULL; size_t numprefix = 0; /* Parse the options. */ diff --git a/src/server.c b/src/server.c index 1001fa4f7..22c81070c 100644 --- a/src/server.c +++ b/src/server.c @@ -2124,6 +2124,10 @@ void beforeSleep(struct aeEventLoop *eventLoop) { if (listLength(server.unblocked_clients)) processUnblockedClients(); + /* Send the invalidation messages to clients participating to the + * client side caching protocol in broadcasting (BCAST) mode. */ + trackingBroadcastInvalidationMessages(); + /* Write the AOF buffer on disk */ flushAppendOnlyFile(0); diff --git a/src/tracking.c b/src/tracking.c index 345c5f1ad..672b886a3 100644 --- a/src/tracking.c +++ b/src/tracking.c @@ -85,6 +85,8 @@ void disableTracking(client *c) { } } raxStop(&ri); + raxFree(c->client_tracking_prefixes); + c->client_tracking_prefixes = NULL; } /* Clear flags and adjust the count. */ @@ -108,6 +110,8 @@ void enableBcastTrackingForPrefix(client *c, char *prefix, size_t plen) { raxInsert(PrefixTable,(unsigned char*)prefix,plen,bs,NULL); } if (raxTryInsert(bs->clients,(unsigned char*)&c,sizeof(c),NULL,NULL)) { + if (c->client_tracking_prefixes == NULL) + c->client_tracking_prefixes = raxNew(); raxInsert(c->client_tracking_prefixes, (unsigned char*)prefix,plen,NULL,NULL); } @@ -121,10 +125,10 @@ void enableBcastTrackingForPrefix(client *c, char *prefix, size_t plen) { * inform it of the condition. Multiple clients can redirect the invalidation * messages to the same client ID. */ void enableTracking(client *c, uint64_t redirect_to, int bcast, robj **prefix, size_t numprefix) { - c->flags |= CLIENT_TRACKING; - c->flags &= ~CLIENT_TRACKING_BROKEN_REDIR; - c->client_tracking_redirection = redirect_to; if (!(c->flags & CLIENT_TRACKING)) server.tracking_clients++; + c->flags |= CLIENT_TRACKING; + c->flags &= ~(CLIENT_TRACKING_BROKEN_REDIR|CLIENT_TRACKING_BCAST); + c->client_tracking_redirection = redirect_to; if (TrackingTable == NULL) { TrackingTable = raxNew(); PrefixTable = raxNew(); @@ -229,10 +233,11 @@ void trackingRememberKeyToBroadcast(char *keyname, size_t keylen) { raxStart(&ri,PrefixTable); raxSeek(&ri,"^",NULL,0); while(raxNext(&ri)) { - if (keylen > ri.key_len) continue; - if (memcmp(ri.key,keyname,ri.key_len) != 0) continue; - bcastState *bs = ri.data; - raxTryInsert(bs->keys,(unsigned char*)keyname,keylen,NULL,NULL); + if (ri.key_len > keylen) continue; + if (ri.key_len != 0 && memcmp(ri.key,keyname,ri.key_len) != 0) + continue; + bcastState *bs = ri.data; + raxTryInsert(bs->keys,(unsigned char*)keyname,keylen,NULL,NULL); } raxStop(&ri); } @@ -362,46 +367,52 @@ void trackingLimitUsedSlots(void) { * notifications to each client in each prefix. */ void trackingBroadcastInvalidationMessages(void) { raxIterator ri, ri2; + + /* Return ASAP if there is nothing to do here. */ + if (TrackingTable == NULL || !server.tracking_clients) return; + raxStart(&ri,PrefixTable); raxSeek(&ri,"^",NULL,0); while(raxNext(&ri)) { bcastState *bs = ri.data; - /* Create the array reply with the list of keys once, then send - * it to all the clients subscribed to this prefix. */ - char buf[32]; - size_t len = ll2string(buf,sizeof(buf),raxSize(bs->keys)); - sds proto = sdsempty(); - proto = sdsMakeRoomFor(proto,raxSize(bs->keys)*15); - proto = sdscatlen(proto,"*",1); - proto = sdscatlen(proto,buf,len); - proto = sdscatlen(proto,"\r\n",2); - raxStart(&ri2,bs->keys); - raxSeek(&ri2,"^",NULL,0); - while(raxNext(&ri2)) { - len = ll2string(buf,sizeof(buf),ri2.key_len); - sds proto = sdsnewlen("$",1); - proto = sdscatlen(proto,ri2.key,ri2.key_len); + if (raxSize(bs->keys)) { + /* Create the array reply with the list of keys once, then send + * it to all the clients subscribed to this prefix. */ + char buf[32]; + size_t len = ll2string(buf,sizeof(buf),raxSize(bs->keys)); + sds proto = sdsempty(); + proto = sdsMakeRoomFor(proto,raxSize(bs->keys)*15); + proto = sdscatlen(proto,"*",1); + proto = sdscatlen(proto,buf,len); proto = sdscatlen(proto,"\r\n",2); - } - raxStop(&ri2); + raxStart(&ri2,bs->keys); + raxSeek(&ri2,"^",NULL,0); + while(raxNext(&ri2)) { + len = ll2string(buf,sizeof(buf),ri2.key_len); + proto = sdscatlen(proto,"$",1); + proto = sdscatlen(proto,buf,len); + proto = sdscatlen(proto,"\r\n",2); + proto = sdscatlen(proto,ri2.key,ri2.key_len); + proto = sdscatlen(proto,"\r\n",2); + } + raxStop(&ri2); - /* Send this array of keys to every client in the list. */ - raxStart(&ri2,bs->clients); - raxSeek(&ri2,"^",NULL,0); - while(raxNext(&ri2)) { - client *c; - memcpy(&c,ri2.key,sizeof(c)); - sendTrackingMessage(c,proto,sdslen(proto),1); - } - raxStop(&ri2); + /* Send this array of keys to every client in the list. */ + raxStart(&ri2,bs->clients); + raxSeek(&ri2,"^",NULL,0); + while(raxNext(&ri2)) { + client *c; + memcpy(&c,ri2.key,sizeof(c)); + sendTrackingMessage(c,proto,sdslen(proto),1); + } + raxStop(&ri2); - /* Clean up: we can remove everything from this state, because we - * want to only track the new keys that will be accumulated starting - * from now. */ - sdsfree(proto); - raxFree(bs->clients); + /* Clean up: we can remove everything from this state, because we + * want to only track the new keys that will be accumulated starting + * from now. */ + sdsfree(proto); + } raxFree(bs->keys); - bs->clients = raxNew(); bs->keys = raxNew(); } raxStop(&ri); From c6cf2ed643da129abe505f16424d3a7019b2fe9d Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 13 Feb 2020 16:58:07 +0100 Subject: [PATCH 0110/1280] Tracking: fix sending messages bug + tracking off bug. --- src/networking.c | 42 ++++++++++++++++++++++-------------------- src/tracking.c | 6 ++++++ 2 files changed, 28 insertions(+), 20 deletions(-) diff --git a/src/networking.c b/src/networking.c index 46534253e..1b4b19645 100644 --- a/src/networking.c +++ b/src/networking.c @@ -2250,37 +2250,39 @@ NULL prefix = zrealloc(prefix,sizeof(robj*)*(numprefix+1)); prefix[numprefix++] = c->argv[j]; } else { + zfree(prefix); addReply(c,shared.syntaxerr); return; } } - /* Make sure options are compatible among each other and with the - * current state of the client. */ - if (!bcast && numprefix) { - addReplyError(c,"PREFIX option requires BCAST mode to be enabled"); - zfree(prefix); - return; - } - - if (c->flags & CLIENT_TRACKING) { - int oldbcast = !!c->flags & CLIENT_TRACKING_BCAST; - if (oldbcast != bcast) { - } - addReplyError(c, - "You can't switch BCAST mode on/off before disabling " - "tracking for this client, and then re-enabling it with " - "a different mode."); - zfree(prefix); - return; - } - /* Options are ok: enable or disable the tracking for this client. */ if (!strcasecmp(c->argv[2]->ptr,"on")) { + /* Before enabling tracking, make sure options are compatible + * among each other and with the current state of the client. */ + if (!bcast && numprefix) { + addReplyError(c, + "PREFIX option requires BCAST mode to be enabled"); + zfree(prefix); + return; + } + + if (c->flags & CLIENT_TRACKING) { + int oldbcast = !!c->flags & CLIENT_TRACKING_BCAST; + if (oldbcast != bcast) { + addReplyError(c, + "You can't switch BCAST mode on/off before disabling " + "tracking for this client, and then re-enabling it with " + "a different mode."); + zfree(prefix); + return; + } + } enableTracking(c,redir,bcast,prefix,numprefix); } else if (!strcasecmp(c->argv[2]->ptr,"off")) { disableTracking(c); } else { + zfree(prefix); addReply(c,shared.syntaxerr); return; } diff --git a/src/tracking.c b/src/tracking.c index 672b886a3..ef5840863 100644 --- a/src/tracking.c +++ b/src/tracking.c @@ -211,6 +211,12 @@ void sendTrackingMessage(client *c, char *keyname, size_t keylen, int proto) { /* We use a static object to speedup things, however we assume * that addReplyPubsubMessage() will not take a reference. */ addReplyPubsubMessage(c,TrackingChannelName,NULL); + } else { + /* If are here, the client is not using RESP3, nor is + * redirecting to another client. We can't send anything to + * it since RESP2 does not support push messages in the same + * connection. */ + return; } /* Send the "value" part, which is the array of keys. */ From 4597ec9b1cdc39431f05fe850f9d15d449a7a3e3 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 14 Feb 2020 14:17:10 +0100 Subject: [PATCH 0111/1280] Tracking: fix behavior when switchinig from normal to BCAST. --- src/tracking.c | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/tracking.c b/src/tracking.c index ef5840863..3333472a4 100644 --- a/src/tracking.c +++ b/src/tracking.c @@ -269,7 +269,17 @@ void trackingInvalidateKey(robj *keyobj) { uint64_t id; memcpy(&id,ri.key,sizeof(id)); client *c = lookupClientByID(id); - if (c == NULL || !(c->flags & CLIENT_TRACKING)) continue; + /* Note that if the client is in BCAST mode, we don't want to + * send invalidation messages that were pending in the case + * previously the client was not in BCAST mode. This can happen if + * TRACKING is enabled normally, and then the client switches to + * BCAST mode. */ + if (c == NULL || + !(c->flags & CLIENT_TRACKING)|| + c->flags & CLIENT_TRACKING_BCAST) + { + continue; + } sendTrackingMessage(c,sdskey,sdslen(sdskey),0); } raxStop(&ri); From 8cb485ad3b612bf3b7d187c6bd06d1c340aebf52 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 14 Feb 2020 14:27:22 +0100 Subject: [PATCH 0112/1280] Tracking: fix operators precedence error in bcast check. --- src/networking.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/networking.c b/src/networking.c index 1b4b19645..69350eed1 100644 --- a/src/networking.c +++ b/src/networking.c @@ -2268,7 +2268,7 @@ NULL } if (c->flags & CLIENT_TRACKING) { - int oldbcast = !!c->flags & CLIENT_TRACKING_BCAST; + int oldbcast = !!(c->flags & CLIENT_TRACKING_BCAST); if (oldbcast != bcast) { addReplyError(c, "You can't switch BCAST mode on/off before disabling " From 0ae05a229a09dfceaedb0bf0610a4c033dcce2e6 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 14 Feb 2020 14:29:00 +0100 Subject: [PATCH 0113/1280] Tracking: first set of tests for the feature. --- tests/unit/tracking.tcl | 66 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 tests/unit/tracking.tcl diff --git a/tests/unit/tracking.tcl b/tests/unit/tracking.tcl new file mode 100644 index 000000000..2058319f7 --- /dev/null +++ b/tests/unit/tracking.tcl @@ -0,0 +1,66 @@ +start_server {tags {"tracking"}} { + # Create a deferred client we'll use to redirect invalidation + # messages to. + set rd1 [redis_deferring_client] + $rd1 client id + set redir [$rd1 read] + $rd1 subscribe __redis__:invalidate + $rd1 read ; # Consume the SUBSCRIBE reply. + + test {Clients are able to enable tracking and redirect it} { + r CLIENT TRACKING on REDIRECT $redir + } {*OK} + + test {The other connection is able to get invalidations} { + r SET a 1 + r GET a + r INCR a + r INCR b ; # This key should not be notified, since it wasn't fetched. + set keys [lindex [$rd1 read] 2] + assert {[llength $keys] == 1} + assert {[lindex $keys 0] eq {a}} + } + + test {The client is now able to disable tracking} { + # Make sure to add a few more keys in the tracking list + # so that we can check for leaks, as a side effect. + r MGET a b c d e f g + r CLIENT TRACKING off + } + + test {Clients can enable the BCAST mode with the empty prefix} { + r CLIENT TRACKING on BCAST REDIRECT $redir + } {*OK*} + + test {The connection gets invalidation messages about all the keys} { + r MSET a 1 b 2 c 3 + set keys [lsort [lindex [$rd1 read] 2]] + assert {$keys eq {a b c}} + } + + test {Clients can enable the BCAST mode with prefixes} { + r CLIENT TRACKING off + r CLIENT TRACKING on BCAST REDIRECT $redir PREFIX a: PREFIX b: + r MULTI + r INCR a:1 + r INCR a:2 + r INCR b:1 + r INCR b:2 + r EXEC + # Because of the internals, we know we are going to receive + # two separated notifications for the two different prefixes. + set keys1 [lsort [lindex [$rd1 read] 2]] + set keys2 [lsort [lindex [$rd1 read] 2]] + set keys [lsort [list {*}$keys1 {*}$keys2]] + assert {$keys eq {a:1 a:2 b:1 b:2}} + } + + test {Adding prefixes to BCAST mode works} { + r CLIENT TRACKING on BCAST REDIRECT $redir PREFIX c: + r INCR c:1234 + set keys [lsort [lindex [$rd1 read] 2]] + assert {$keys eq {c:1234}} + } + + $rd1 close +} From 53aa39c059880296e5e8ba759a04794400ca8f64 Mon Sep 17 00:00:00 2001 From: Itamar Haber Date: Fri, 14 Feb 2020 17:13:58 +0200 Subject: [PATCH 0114/1280] Fixes segfault on calling trackingGetTotalKeys ... with CSC disabled --- src/tracking.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/tracking.c b/src/tracking.c index 3333472a4..7179a54f8 100644 --- a/src/tracking.c +++ b/src/tracking.c @@ -441,5 +441,6 @@ uint64_t trackingGetTotalItems(void) { } uint64_t trackingGetTotalKeys(void) { + if (TrackingTable == NULL) return 0; return raxSize(TrackingTable); } From 3b031b16004c7575d4315c0bb60f3cba97c83817 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 14 Feb 2020 18:22:25 +0100 Subject: [PATCH 0115/1280] Signal key as modified when expired on-access. This fixes WATCH and client side caching with keys expiring because of a synchronous access and not because of background expiring. --- src/db.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/db.c b/src/db.c index c80524e94..8a0242d9e 100644 --- a/src/db.c +++ b/src/db.c @@ -1296,8 +1296,10 @@ int expireIfNeeded(redisDb *db, robj *key) { propagateExpire(db,key,server.lazyfree_lazy_expire); notifyKeyspaceEvent(NOTIFY_EXPIRED, "expired",key,db->id); - return server.lazyfree_lazy_expire ? dbAsyncDelete(db,key) : - dbSyncDelete(db,key); + int retval = server.lazyfree_lazy_expire ? dbAsyncDelete(db,key) : + dbSyncDelete(db,key); + if (retval) signalModifiedKey(db,key); + return retval; } /* ----------------------------------------------------------------------------- From 7f07fed499b057580b26005fc7856c69b9651738 Mon Sep 17 00:00:00 2001 From: Madelyn Olson Date: Sun, 16 Feb 2020 05:16:51 -0800 Subject: [PATCH 0116/1280] Minor CSC fixes and fixed documentation --- src/networking.c | 11 ++++++++--- src/tracking.c | 22 ++++++++-------------- 2 files changed, 16 insertions(+), 17 deletions(-) diff --git a/src/networking.c b/src/networking.c index 69350eed1..dad61904d 100644 --- a/src/networking.c +++ b/src/networking.c @@ -2044,7 +2044,7 @@ void clientCommand(client *c) { "REPLY (on|off|skip) -- Control the replies sent to the current connection.", "SETNAME -- Assign the name to the current connection.", "UNBLOCK [TIMEOUT|ERROR] -- Unblock the specified blocked client.", -"TRACKING (on|off) [REDIRECT ] -- Enable client keys tracking for client side caching.", +"TRACKING (on|off) [REDIRECT ] [BCAST] [PREFIX first] [PREFIX second] ... -- Enable client keys tracking for client side caching.", "GETREDIR -- Return the client ID we are redirecting to when tracking is enabled.", NULL }; @@ -2234,17 +2234,22 @@ NULL if (!strcasecmp(c->argv[j]->ptr,"redirect") && moreargs) { j++; if (getLongLongFromObjectOrReply(c,c->argv[j],&redir,NULL) != - C_OK) return; + C_OK) + { + zfree(prefix); + return; + } /* We will require the client with the specified ID to exist * right now, even if it is possible that it gets disconnected * later. Still a valid sanity check. */ if (lookupClientByID(redir) == NULL) { addReplyError(c,"The client ID you want redirect to " "does not exist"); + zfree(prefix); return; } } else if (!strcasecmp(c->argv[j]->ptr,"bcast")) { - bcast++; + bcast = 1; } else if (!strcasecmp(c->argv[j]->ptr,"prefix") && moreargs) { j++; prefix = zrealloc(prefix,sizeof(robj*)*(numprefix+1)); diff --git a/src/tracking.c b/src/tracking.c index 7179a54f8..619148f2f 100644 --- a/src/tracking.c +++ b/src/tracking.c @@ -44,7 +44,7 @@ rax *TrackingTable = NULL; rax *PrefixTable = NULL; uint64_t TrackingTableTotalItems = 0; /* Total number of IDs stored across - the whole tracking table. This givesn + the whole tracking table. This gives an hint about the total memory we are using server side for CSC. */ robj *TrackingChannelName; @@ -145,9 +145,9 @@ void enableTracking(client *c, uint64_t redirect_to, int bcast, robj **prefix, s } } -/* This function is called after the excution of a readonly command in the +/* This function is called after the execution of a readonly command in the * case the client 'c' has keys tracking enabled. It will populate the - * tracking ivalidation table according to the keys the user fetched, so that + * tracking invalidation table according to the keys the user fetched, so that * Redis will know what are the clients that should receive an invalidation * message with certain groups of keys are modified. */ void trackingRememberKeys(client *c) { @@ -292,19 +292,12 @@ void trackingInvalidateKey(robj *keyobj) { } /* This function is called when one or all the Redis databases are flushed - * (dbid == -1 in case of FLUSHALL). Caching slots are not specific for - * each DB but are global: currently what we do is sending a special + * (dbid == -1 in case of FLUSHALL). Caching keys are not specific for + * each DB but are global: currently what we do is send a special * notification to clients with tracking enabled, invalidating the caching - * slot "-1", which means, "all the keys", in order to avoid flooding clients + * key "", which means, "all the keys", in order to avoid flooding clients * with many invalidation messages for all the keys they may hold. - * - * However trying to flush the tracking table here is very costly: - * we need scanning 16 million caching slots in the table to check - * if they are used, this introduces a big delay. So what we do is to really - * flush the table in the case of FLUSHALL. When a FLUSHDB is called instead - * we just send the invalidation message to all the clients, but don't - * flush the table: it will slowly get garbage collected as more keys - * are modified in the used caching slots. */ + */ void freeTrackingRadixTree(void *rt) { raxFree(rt); } @@ -325,6 +318,7 @@ void trackingInvalidateKeysOnFlush(int dbid) { /* In case of FLUSHALL, reclaim all the memory used by tracking. */ if (dbid == -1 && TrackingTable) { raxFreeWithCallback(TrackingTable,freeTrackingRadixTree); + TrackingTable = raxNew(); TrackingTableTotalItems = 0; } } From 651a1b2262df668ad9d8458d691613cf001c07cc Mon Sep 17 00:00:00 2001 From: Madelyn Olson Date: Sun, 16 Feb 2020 05:41:39 -0800 Subject: [PATCH 0117/1280] Give an error message if you specify redirect twice --- src/networking.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/networking.c b/src/networking.c index dad61904d..5b1229fde 100644 --- a/src/networking.c +++ b/src/networking.c @@ -2233,6 +2233,13 @@ NULL if (!strcasecmp(c->argv[j]->ptr,"redirect") && moreargs) { j++; + if (redir != 0) { + addReplyError(c,"A client can only redirect to a single " + "other client"); + zfree(prefix); + return; + } + if (getLongLongFromObjectOrReply(c,c->argv[j],&redir,NULL) != C_OK) { From 4006d36600c9a6951da8e54571c5c441f10ba980 Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Sun, 16 Feb 2020 15:43:19 +0200 Subject: [PATCH 0118/1280] module api docs for aux_save and aux_load --- src/module.c | 6 ++++++ src/rdb.c | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/module.c b/src/module.c index aae15d74b..69101161d 100644 --- a/src/module.c +++ b/src/module.c @@ -3529,6 +3529,8 @@ void moduleTypeNameByID(char *name, uint64_t moduleid) { * // Optional fields * .digest = myType_DigestCallBack, * .mem_usage = myType_MemUsageCallBack, + * .aux_load = myType_AuxRDBLoadCallBack, + * .aux_save = myType_AuxRDBSaveCallBack, * } * * * **rdb_load**: A callback function pointer that loads data from RDB files. @@ -3536,6 +3538,10 @@ void moduleTypeNameByID(char *name, uint64_t moduleid) { * * **aof_rewrite**: A callback function pointer that rewrites data as commands. * * **digest**: A callback function pointer that is used for `DEBUG DIGEST`. * * **free**: A callback function pointer that can free a type value. + * * **aux_save**: A callback function pointer that saves out of keyspace data to RDB files. + * 'when' argument is either REDISMODULE_AUX_BEFORE_RDB or REDISMODULE_AUX_AFTER_RDB. + * * **aux_load**: A callback function pointer that loads out of keyspace data from RDB files. + * Similar to aux_save, returns REDISMODULE_OK on success, and ERR otherwise. * * The **digest* and **mem_usage** methods should currently be omitted since * they are not yet implemented inside the Redis modules core. diff --git a/src/rdb.c b/src/rdb.c index 61265433d..cbcea96c6 100644 --- a/src/rdb.c +++ b/src/rdb.c @@ -2195,7 +2195,7 @@ int rdbLoadRio(rio *rdb, int rdbflags, rdbSaveInfo *rsi) { io.ver = 2; /* Call the rdb_load method of the module providing the 10 bit * encoding version in the lower 10 bits of the module ID. */ - if (mt->aux_load(&io,moduleid&1023, when) || io.error) { + if (mt->aux_load(&io,moduleid&1023, when) != REDISMODULE_OK || io.error) { moduleTypeNameByID(name,moduleid); serverLog(LL_WARNING,"The RDB file contains module AUX data for the module type '%s', that the responsible module is not able to load. Check for modules log above for additional clues.", name); exit(1); From 3efa9dd5c020529ad6c3dea3eb2b5484e08e9467 Mon Sep 17 00:00:00 2001 From: hwware Date: Mon, 17 Feb 2020 23:40:24 -0500 Subject: [PATCH 0119/1280] add missing subcommand description for debug oom --- src/debug.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/debug.c b/src/debug.c index b910d2d2d..dd96ad416 100644 --- a/src/debug.c +++ b/src/debug.c @@ -363,6 +363,7 @@ void debugCommand(client *c) { "LOADAOF -- Flush the AOF buffers on disk and reload the AOF in memory.", "LUA-ALWAYS-REPLICATE-COMMANDS <0|1> -- Setting it to 1 makes Lua replication defaulting to replicating single commands, without the script having to enable effects replication.", "OBJECT -- Show low level info about key and associated value.", +"OOM -- Crash the server simulating an out-of-memory error.", "PANIC -- Crash the server simulating a panic.", "POPULATE [prefix] [size] -- Create string keys named key:. If a prefix is specified is used instead of the 'key' prefix.", "RELOAD -- Save the RDB on disk and reload it back in memory.", From d3a0fc70c374fa38d499639edee4d05feeb2d987 Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Tue, 18 Feb 2020 16:19:52 +0200 Subject: [PATCH 0120/1280] Defrag big lists in portions to avoid latency and freeze When active defrag kicks in and finds a big list, it will create a bookmark to a node so that it is able to resume iteration from that node later. The quicklist manages that bookmark, and updates it in case that node is deleted. This will increase memory usage only on lists of over 1000 (see active-defrag-max-scan-fields) quicklist nodes (1000 ziplists, not 1000 items) by 16 bytes. In 32 bit build, this change reduces the maximum effective config of list-compress-depth and list-max-ziplist-size (from 32767 to 8191) --- src/defrag.c | 96 +++++++++++++++------- src/quicklist.c | 150 ++++++++++++++++++++++++++++++++++- src/quicklist.h | 46 ++++++++++- tests/unit/memefficiency.tcl | 92 +++++++++++++++++++++ 4 files changed, 350 insertions(+), 34 deletions(-) diff --git a/src/defrag.c b/src/defrag.c index 04e57955b..e729297a5 100644 --- a/src/defrag.c +++ b/src/defrag.c @@ -5,8 +5,8 @@ * We do that by scanning the keyspace and for each pointer we have, we can try to * ask the allocator if moving it to a new address will help reduce fragmentation. * - * Copyright (c) 2017, Oran Agra - * Copyright (c) 2017, Redis Labs, Inc + * Copyright (c) 2020, Oran Agra + * Copyright (c) 2020, Redis Labs, Inc * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -408,25 +408,32 @@ dictEntry* replaceSateliteDictKeyPtrAndOrDefragDictEntry(dict *d, sds oldkey, sd return NULL; } -long activeDefragQuickListNodes(quicklist *ql) { - quicklistNode *node = ql->head, *newnode; +long activeDefragQuickListNode(quicklist *ql, quicklistNode **node_ref) { + quicklistNode *newnode, *node = *node_ref; long defragged = 0; unsigned char *newzl; + if ((newnode = activeDefragAlloc(node))) { + if (newnode->prev) + newnode->prev->next = newnode; + else + ql->head = newnode; + if (newnode->next) + newnode->next->prev = newnode; + else + ql->tail = newnode; + *node_ref = node = newnode; + defragged++; + } + if ((newzl = activeDefragAlloc(node->zl))) + defragged++, node->zl = newzl; + return defragged; +} + +long activeDefragQuickListNodes(quicklist *ql) { + quicklistNode *node = ql->head; + long defragged = 0; while (node) { - if ((newnode = activeDefragAlloc(node))) { - if (newnode->prev) - newnode->prev->next = newnode; - else - ql->head = newnode; - if (newnode->next) - newnode->next->prev = newnode; - else - ql->tail = newnode; - node = newnode; - defragged++; - } - if ((newzl = activeDefragAlloc(node->zl))) - defragged++, node->zl = newzl; + defragged += activeDefragQuickListNode(ql, &node); node = node->next; } return defragged; @@ -440,12 +447,48 @@ void defragLater(redisDb *db, dictEntry *kde) { listAddNodeTail(db->defrag_later, key); } -long scanLaterList(robj *ob) { +/* returns 0 if no more work needs to be been done, and 1 if time is up and more work is needed. */ +long scanLaterList(robj *ob, unsigned long *cursor, long long endtime, long long *defragged) { quicklist *ql = ob->ptr; + quicklistNode *node; + long iterations = 0; + int bookmark_failed = 0; if (ob->type != OBJ_LIST || ob->encoding != OBJ_ENCODING_QUICKLIST) return 0; - server.stat_active_defrag_scanned+=ql->len; - return activeDefragQuickListNodes(ql); + + if (*cursor == 0) { + /* if cursor is 0, we start new iteration */ + node = ql->head; + } else { + node = quicklistBookmarkFind(ql, "_AD"); + if (!node) { + /* if the bookmark was deleted, it means we reached the end. */ + *cursor = 0; + return 0; + } + node = node->next; + } + + (*cursor)++; + while (node) { + (*defragged) += activeDefragQuickListNode(ql, &node); + server.stat_active_defrag_scanned++; + if (++iterations > 128 && !bookmark_failed) { + if (ustime() > endtime) { + if (!quicklistBookmarkCreate(&ql, "_AD", node)) { + bookmark_failed = 1; + } else { + ob->ptr = ql; /* bookmark creation may have re-allocated the quicklist */ + return 1; + } + } + iterations = 0; + } + node = node->next; + } + quicklistBookmarkDelete(ql, "_AD"); + *cursor = 0; + return bookmark_failed? 1: 0; } typedef struct { @@ -638,7 +681,8 @@ int scanLaterStraemListpacks(robj *ob, unsigned long *cursor, long long endtime, void *newdata = activeDefragAlloc(ri.data); if (newdata) raxSetData(ri.node, ri.data=newdata), (*defragged)++; - if (++iterations > 16) { + server.stat_active_defrag_scanned++; + if (++iterations > 128) { if (ustime() > endtime) { serverAssert(ri.key_len==sizeof(last)); memcpy(last,ri.key,ri.key_len); @@ -900,8 +944,7 @@ int defragLaterItem(dictEntry *de, unsigned long *cursor, long long endtime) { if (de) { robj *ob = dictGetVal(de); if (ob->type == OBJ_LIST) { - server.stat_active_defrag_hits += scanLaterList(ob); - *cursor = 0; /* list has no scan, we must finish it in one go */ + return scanLaterList(ob, cursor, endtime, &server.stat_active_defrag_hits); } else if (ob->type == OBJ_SET) { server.stat_active_defrag_hits += scanLaterSet(ob, cursor); } else if (ob->type == OBJ_ZSET) { @@ -961,11 +1004,6 @@ int defragLaterStep(redisDb *db, long long endtime) { if (defragLaterItem(de, &defrag_later_cursor, endtime)) quit = 1; /* time is up, we didn't finish all the work */ - /* Don't start a new BIG key in this loop, this is because the - * next key can be a list, and scanLaterList must be done in once cycle */ - if (!defrag_later_cursor) - quit = 1; - /* Once in 16 scan iterations, 512 pointer reallocations, or 64 fields * (if we have a lot of pointers in one hash bucket, or rehashing), * check if we reached the time limit. */ diff --git a/src/quicklist.c b/src/quicklist.c index 7b5484116..ae183ffd8 100644 --- a/src/quicklist.c +++ b/src/quicklist.c @@ -70,6 +70,12 @@ static const size_t optimization_level[] = {4096, 8192, 16384, 32768, 65536}; } while (0); #endif +/* Bookmarks forward declarations */ +#define QL_MAX_BM ((1 << QL_BM_BITS)-1) +quicklistBookmark *_quicklistBookmarkFindByName(quicklist *ql, const char *name); +quicklistBookmark *_quicklistBookmarkFindByNode(quicklist *ql, quicklistNode *node); +void _quicklistBookmarkDelete(quicklist *ql, quicklistBookmark *bm); + /* Simple way to give quicklistEntry structs default values with one call. */ #define initEntry(e) \ do { \ @@ -100,10 +106,11 @@ quicklist *quicklistCreate(void) { quicklist->count = 0; quicklist->compress = 0; quicklist->fill = -2; + quicklist->bookmark_count = 0; return quicklist; } -#define COMPRESS_MAX (1 << 16) +#define COMPRESS_MAX (1 << QL_COMP_BITS) void quicklistSetCompressDepth(quicklist *quicklist, int compress) { if (compress > COMPRESS_MAX) { compress = COMPRESS_MAX; @@ -113,7 +120,7 @@ void quicklistSetCompressDepth(quicklist *quicklist, int compress) { quicklist->compress = compress; } -#define FILL_MAX (1 << 15) +#define FILL_MAX (1 << (QL_FILL_BITS-1)) void quicklistSetFill(quicklist *quicklist, int fill) { if (fill > FILL_MAX) { fill = FILL_MAX; @@ -169,6 +176,7 @@ void quicklistRelease(quicklist *quicklist) { quicklist->len--; current = next; } + quicklistBookmarksClear(quicklist); zfree(quicklist); } @@ -578,6 +586,15 @@ quicklist *quicklistCreateFromZiplist(int fill, int compress, REDIS_STATIC void __quicklistDelNode(quicklist *quicklist, quicklistNode *node) { + /* Update the bookmark if any */ + quicklistBookmark *bm = _quicklistBookmarkFindByNode(quicklist, node); + if (bm) { + bm->node = node->next; + /* if the bookmark was to the last node, delete it. */ + if (!bm->node) + _quicklistBookmarkDelete(quicklist, bm); + } + if (node->next) node->next->prev = node->prev; if (node->prev) @@ -1410,6 +1427,87 @@ void quicklistPush(quicklist *quicklist, void *value, const size_t sz, } } +/* Create or update a bookmark in the list which will be updated to the next node + * automatically when the one referenced gets deleted. + * Returns 1 on success (creation of new bookmark or override of an existing one). + * Returns 0 on failure (reached the maximum supported number of bookmarks). + * NOTE: use short simple names, so that string compare on find is quick. + * NOTE: bookmakrk creation may re-allocate the quicklist, so the input pointer + may change and it's the caller responsibilty to update the reference. + */ +int quicklistBookmarkCreate(quicklist **ql_ref, const char *name, quicklistNode *node) { + quicklist *ql = *ql_ref; + if (ql->bookmark_count >= QL_MAX_BM) + return 0; + quicklistBookmark *bm = _quicklistBookmarkFindByName(ql, name); + if (bm) { + bm->node = node; + return 1; + } + ql = zrealloc(ql, sizeof(quicklist) + (ql->bookmark_count+1) * sizeof(quicklistBookmark)); + *ql_ref = ql; + ql->bookmarks[ql->bookmark_count].node = node; + ql->bookmarks[ql->bookmark_count].name = zstrdup(name); + ql->bookmark_count++; + return 1; +} + +/* Find the quicklist node referenced by a named bookmark. + * When the bookmarked node is deleted the bookmark is updated to the next node, + * and if that's the last node, the bookmark is deleted (so find returns NULL). */ +quicklistNode *quicklistBookmarkFind(quicklist *ql, const char *name) { + quicklistBookmark *bm = _quicklistBookmarkFindByName(ql, name); + if (!bm) return NULL; + return bm->node; +} + +/* Delete a named bookmark. + * returns 0 if bookmark was not found, and 1 if deleted. + * Note that the bookmark memory is not freed yet, and is kept for future use. */ +int quicklistBookmarkDelete(quicklist *ql, const char *name) { + quicklistBookmark *bm = _quicklistBookmarkFindByName(ql, name); + if (!bm) + return 0; + _quicklistBookmarkDelete(ql, bm); + return 1; +} + +quicklistBookmark *_quicklistBookmarkFindByName(quicklist *ql, const char *name) { + unsigned i; + for (i=0; ibookmark_count; i++) { + if (!strcmp(ql->bookmarks[i].name, name)) { + return &ql->bookmarks[i]; + } + } + return NULL; +} + +quicklistBookmark *_quicklistBookmarkFindByNode(quicklist *ql, quicklistNode *node) { + unsigned i; + for (i=0; ibookmark_count; i++) { + if (ql->bookmarks[i].node == node) { + return &ql->bookmarks[i]; + } + } + return NULL; +} + +void _quicklistBookmarkDelete(quicklist *ql, quicklistBookmark *bm) { + int index = bm - ql->bookmarks; + zfree(bm->name); + ql->bookmark_count--; + memmove(bm, bm+1, (ql->bookmark_count - index)* sizeof(*bm)); + /* NOTE: We do not shrink (realloc) the quicklist yet (to avoid resonance, + * it may be re-used later (a call to realloc may NOP). */ +} + +void quicklistBookmarksClear(quicklist *ql) { + while (ql->bookmark_count) + zfree(ql->bookmarks[--ql->bookmark_count].name); + /* NOTE: We do not shrink (realloc) the quick list. main use case for this + * function is just before releasing the allocation. */ +} + /* The rest of this file is test cases and test helpers. */ #ifdef REDIS_TEST #include @@ -2641,6 +2739,54 @@ int quicklistTest(int argc, char *argv[]) { printf("Compressions: %0.2f seconds.\n", (float)(stop - start) / 1000); printf("\n"); + TEST("bookmark get updated to next item") { + quicklist *ql = quicklistNew(1, 0); + quicklistPushTail(ql, "1", 1); + quicklistPushTail(ql, "2", 1); + quicklistPushTail(ql, "3", 1); + quicklistPushTail(ql, "4", 1); + quicklistPushTail(ql, "5", 1); + assert(ql->len==5); + /* add two bookmarks, one pointing to the node before the last. */ + assert(quicklistBookmarkCreate(&ql, "_dummy", ql->head->next)); + assert(quicklistBookmarkCreate(&ql, "_test", ql->tail->prev)); + /* test that the bookmark returns the right node, delete it and see that the bookmark points to the last node */ + assert(quicklistBookmarkFind(ql, "_test") == ql->tail->prev); + assert(quicklistDelRange(ql, -2, 1)); + assert(quicklistBookmarkFind(ql, "_test") == ql->tail); + /* delete the last node, and see that the bookmark was deleted. */ + assert(quicklistDelRange(ql, -1, 1)); + assert(quicklistBookmarkFind(ql, "_test") == NULL); + /* test that other bookmarks aren't affected */ + assert(quicklistBookmarkFind(ql, "_dummy") == ql->head->next); + assert(quicklistBookmarkFind(ql, "_missing") == NULL); + assert(ql->len==3); + quicklistBookmarksClear(ql); /* for coverage */ + assert(quicklistBookmarkFind(ql, "_dummy") == NULL); + quicklistRelease(ql); + } + + TEST("bookmark limit") { + int i; + quicklist *ql = quicklistNew(1, 0); + quicklistPushHead(ql, "1", 1); + for (i=0; ihead)); + /* when all bookmarks are used, creation fails */ + assert(!quicklistBookmarkCreate(&ql, "_test", ql->head)); + /* delete one and see that we can now create another */ + assert(quicklistBookmarkDelete(ql, "0")); + assert(quicklistBookmarkCreate(&ql, "_test", ql->head)); + /* delete one and see that the rest survive */ + assert(quicklistBookmarkDelete(ql, "_test")); + for (i=1; ihead); + /* make sure the deleted ones are indeed gone */ + assert(!quicklistBookmarkFind(ql, "0")); + assert(!quicklistBookmarkFind(ql, "_test")); + quicklistRelease(ql); + } + if (!err) printf("ALL TESTS PASSED!\n"); else diff --git a/src/quicklist.h b/src/quicklist.h index a7e27a2dd..8b553c119 100644 --- a/src/quicklist.h +++ b/src/quicklist.h @@ -28,6 +28,8 @@ * POSSIBILITY OF SUCH DAMAGE. */ +#include // for UINTPTR_MAX + #ifndef __QUICKLIST_H__ #define __QUICKLIST_H__ @@ -64,19 +66,51 @@ typedef struct quicklistLZF { char compressed[]; } quicklistLZF; +/* Bookmarks are padded with realloc at the end of of the quicklist struct. + * They should only be used for very big lists if thousands of nodes were the + * excess memory usage is negligible, and there's a real need to iterate on them + * in portions. + * When not used, they don't add any memory overhead, but when used and then + * deleted, some overhead remains (to avoid resonance). + * The number of bookmarks used should be kept to minimum since it also adds + * overhead on node deletion (searching for a bookmark to update). */ +typedef struct quicklistBookmark { + quicklistNode *node; + char *name; +} quicklistBookmark; + +#if UINTPTR_MAX == 0xffffffff +/* 32-bit */ +# define QL_FILL_BITS 14 +# define QL_COMP_BITS 14 +# define QL_BM_BITS 4 +#elif UINTPTR_MAX == 0xffffffffffffffff +/* 64-bit */ +# define QL_FILL_BITS 16 +# define QL_COMP_BITS 16 +# define QL_BM_BITS 4 /* we can encode more, but we rather limit the user + since they cause performance degradation. */ +#else +# error unknown arch bits count +#endif + /* quicklist is a 40 byte struct (on 64-bit systems) describing a quicklist. * 'count' is the number of total entries. * 'len' is the number of quicklist nodes. * 'compress' is: -1 if compression disabled, otherwise it's the number * of quicklistNodes to leave uncompressed at ends of quicklist. - * 'fill' is the user-requested (or default) fill factor. */ + * 'fill' is the user-requested (or default) fill factor. + * 'bookmakrs are an optional feature that is used by realloc this struct, + * so that they don't consume memory when not used. */ typedef struct quicklist { quicklistNode *head; quicklistNode *tail; unsigned long count; /* total count of all entries in all ziplists */ unsigned long len; /* number of quicklistNodes */ - int fill : 16; /* fill factor for individual nodes */ - unsigned int compress : 16; /* depth of end nodes not to compress;0=off */ + int fill : QL_FILL_BITS; /* fill factor for individual nodes */ + unsigned int compress : QL_COMP_BITS; /* depth of end nodes not to compress;0=off */ + unsigned int bookmark_count: QL_BM_BITS; + quicklistBookmark bookmarks[]; } quicklist; typedef struct quicklistIter { @@ -158,6 +192,12 @@ unsigned long quicklistCount(const quicklist *ql); int quicklistCompare(unsigned char *p1, unsigned char *p2, int p2_len); size_t quicklistGetLzf(const quicklistNode *node, void **data); +/* bookmarks */ +int quicklistBookmarkCreate(quicklist **ql_ref, const char *name, quicklistNode *node); +int quicklistBookmarkDelete(quicklist *ql, const char *name); +quicklistNode *quicklistBookmarkFind(quicklist *ql, const char *name); +void quicklistBookmarksClear(quicklist *ql); + #ifdef REDIS_TEST int quicklistTest(int argc, char *argv[]); #endif diff --git a/tests/unit/memefficiency.tcl b/tests/unit/memefficiency.tcl index d152e212c..ec80c7384 100644 --- a/tests/unit/memefficiency.tcl +++ b/tests/unit/memefficiency.tcl @@ -209,5 +209,97 @@ start_server {tags {"defrag"}} { assert {$digest eq $newdigest} r save ;# saving an rdb iterates over all the data / pointers } {OK} + + test "Active defrag big list" { + r flushdb + r config resetstat + r config set save "" ;# prevent bgsave from interfereing with save below + r config set hz 100 + r config set activedefrag no + r config set active-defrag-max-scan-fields 1000 + r config set active-defrag-threshold-lower 5 + r config set active-defrag-cycle-min 65 + r config set active-defrag-cycle-max 75 + r config set active-defrag-ignore-bytes 2mb + r config set maxmemory 0 + r config set list-max-ziplist-size 5 ;# list of 500k items will have 100k quicklist nodes + + # create big keys with 10k items + set rd [redis_deferring_client] + + set expected_frag 1.7 + # add a mass of list nodes to two lists (allocations are interlaced) + set val [string repeat A 100] ;# 5 items of 100 bytes puts us in the 640 bytes bin, which has 32 regs, so high potential for fragmentation + for {set j 0} {$j < 500000} {incr j} { + $rd lpush biglist1 $val + $rd lpush biglist2 $val + } + for {set j 0} {$j < 500000} {incr j} { + $rd read ; # Discard replies + $rd read ; # Discard replies + } + + # create some fragmentation + r del biglist2 + + # start defrag + after 120 ;# serverCron only updates the info once in 100ms + set frag [s allocator_frag_ratio] + if {$::verbose} { + puts "frag $frag" + } + + assert {$frag >= $expected_frag} + r config set latency-monitor-threshold 5 + r latency reset + + set digest [r debug digest] + catch {r config set activedefrag yes} e + if {![string match {DISABLED*} $e]} { + # wait for the active defrag to start working (decision once a second) + wait_for_condition 50 100 { + [s active_defrag_running] ne 0 + } else { + fail "defrag not started." + } + + # wait for the active defrag to stop working + wait_for_condition 500 100 { + [s active_defrag_running] eq 0 + } else { + after 120 ;# serverCron only updates the info once in 100ms + puts [r info memory] + puts [r info stats] + puts [r memory malloc-stats] + fail "defrag didn't stop." + } + + # test the the fragmentation is lower + after 120 ;# serverCron only updates the info once in 100ms + set frag [s allocator_frag_ratio] + set max_latency 0 + foreach event [r latency latest] { + lassign $event eventname time latency max + if {$eventname == "active-defrag-cycle"} { + set max_latency $max + } + } + if {$::verbose} { + puts "frag $frag" + puts "max latency $max_latency" + puts [r latency latest] + puts [r latency history active-defrag-cycle] + } + assert {$frag < 1.1} + # due to high fragmentation, 100hz, and active-defrag-cycle-max set to 75, + # we expect max latency to be not much higher than 7.5ms + assert {$max_latency <= 12} + } + # verify the data isn't corrupted or changed + set newdigest [r debug digest] + assert {$digest eq $newdigest} + r save ;# saving an rdb iterates over all the data / pointers + r del biglist1 ;# coverage for quicklistBookmarksClear + } {1} } } From e3b50b643206985d47d2554373954d9232752cd8 Mon Sep 17 00:00:00 2001 From: Guy Benoish Date: Wed, 19 Feb 2020 08:24:20 +0530 Subject: [PATCH 0121/1280] XGROUP DESTROY should unblock XREADGROUP with -NOGROUP --- src/t_stream.c | 2 ++ tests/unit/type/stream-cgroups.tcl | 9 +++++++++ 2 files changed, 11 insertions(+) diff --git a/src/t_stream.c b/src/t_stream.c index 0f0f97a1d..900fa3a17 100644 --- a/src/t_stream.c +++ b/src/t_stream.c @@ -1850,6 +1850,8 @@ NULL server.dirty++; notifyKeyspaceEvent(NOTIFY_STREAM,"xgroup-destroy", c->argv[2],c->db->id); + /* We want to unblock any XREADGROUP consumers with -NOGROUP. */ + signalKeyAsReady(c->db,c->argv[2]); } else { addReply(c,shared.czero); } diff --git a/tests/unit/type/stream-cgroups.tcl b/tests/unit/type/stream-cgroups.tcl index a59e168ef..072ed14d6 100644 --- a/tests/unit/type/stream-cgroups.tcl +++ b/tests/unit/type/stream-cgroups.tcl @@ -161,6 +161,15 @@ start_server { assert {[$rd read] == {}} ;# before the fix, client didn't even block, but was served synchronously with {mystream {}} } + test {XGROUP DESTROY should unblock XREADGROUP with -NOGROUP} { + r del mystream + r XGROUP CREATE mystream mygroup $ MKSTREAM + set rd [redis_deferring_client] + $rd XREADGROUP GROUP mygroup Alice BLOCK 100 STREAMS mystream ">" + r XGROUP DESTROY mystream mygroup + assert_error "*NOGROUP*" {$rd read} + } + test {XCLAIM can claim PEL items from another consumer} { # Add 3 items into the stream, and create a consumer group r del mystream From c64fa84e08fd8fe3560fd3c907986af50a9c5791 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 19 Feb 2020 19:00:29 +0100 Subject: [PATCH 0122/1280] Tracking: fix max-keys configuration directive. --- src/config.c | 2 +- src/server.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/config.c b/src/config.c index b2e5fc12e..d55d1f8b5 100644 --- a/src/config.c +++ b/src/config.c @@ -2160,7 +2160,6 @@ standardConfig configs[] = { createIntConfig("list-compress-depth", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.list_compress_depth, 0, INTEGER_CONFIG, NULL, NULL), createIntConfig("rdb-key-save-delay", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.rdb_key_save_delay, 0, INTEGER_CONFIG, NULL, NULL), createIntConfig("key-load-delay", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.key_load_delay, 0, INTEGER_CONFIG, NULL, NULL), - createIntConfig("tracking-table-max-fill", NULL, MODIFIABLE_CONFIG, 0, 100, server.tracking_table_max_keys, 1000000, INTEGER_CONFIG, NULL, NULL), /* Default: 10% tracking table max number of keys tracked. */ createIntConfig("active-expire-effort", NULL, MODIFIABLE_CONFIG, 1, 10, server.active_expire_effort, 1, INTEGER_CONFIG, NULL, NULL), /* From 1 to 10. */ createIntConfig("hz", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.config_hz, CONFIG_DEFAULT_HZ, INTEGER_CONFIG, NULL, updateHZ), createIntConfig("min-replicas-to-write", "min-slaves-to-write", MODIFIABLE_CONFIG, 0, INT_MAX, server.repl_min_slaves_to_write, 0, INTEGER_CONFIG, NULL, updateGoodSlaves), @@ -2195,6 +2194,7 @@ standardConfig configs[] = { createSizeTConfig("stream-node-max-bytes", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.stream_node_max_bytes, 4096, MEMORY_CONFIG, NULL, NULL), createSizeTConfig("zset-max-ziplist-value", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.zset_max_ziplist_value, 64, MEMORY_CONFIG, NULL, NULL), createSizeTConfig("hll-sparse-max-bytes", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.hll_sparse_max_bytes, 3000, MEMORY_CONFIG, NULL, NULL), + createSizeTConfig("tracking-table-max-keys", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.tracking_table_max_keys, 1000000, INTEGER_CONFIG, NULL, NULL), /* Default: 1 million keys max. */ /* Other configs */ createTimeTConfig("repl-backlog-ttl", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.repl_backlog_time_limit, 60*60, INTEGER_CONFIG, NULL, NULL), /* Default: 1 hour */ diff --git a/src/server.h b/src/server.h index 439bbc393..bb40ca4e1 100644 --- a/src/server.h +++ b/src/server.h @@ -1309,7 +1309,7 @@ struct redisServer { list *ready_keys; /* List of readyList structures for BLPOP & co */ /* Client side caching. */ unsigned int tracking_clients; /* # of clients with tracking enabled.*/ - int tracking_table_max_keys; /* Max number of keys in tracking table. */ + size_t tracking_table_max_keys; /* Max number of keys in tracking table. */ /* Sort parameters - qsort_r() is only available under BSD so we * have to take this state global, in order to pass it to sortCompare() */ int sort_desc; From fbcdb676ca6218886ab90ea860a59e47406fb2e4 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 21 Feb 2020 13:48:43 +0100 Subject: [PATCH 0123/1280] Test is more complex now, increase default timeout. --- tests/test_helper.tcl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_helper.tcl b/tests/test_helper.tcl index cb7e4e328..b266bc56d 100644 --- a/tests/test_helper.tcl +++ b/tests/test_helper.tcl @@ -87,7 +87,7 @@ set ::file ""; # If set, runs only the tests in this comma separated list set ::curfile ""; # Hold the filename of the current suite set ::accurate 0; # If true runs fuzz tests with more iterations set ::force_failure 0 -set ::timeout 600; # 10 minutes without progresses will quit the test. +set ::timeout 1200; # 20 minutes without progresses will quit the test. set ::last_progress [clock seconds] set ::active_servers {} ; # Pids of active Redis instances. set ::dont_clean 0 From 55bd09593df06afd8eca3512347b4c2169ef5fe3 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 21 Feb 2020 17:08:45 +0100 Subject: [PATCH 0124/1280] Test engine: better tracking of what workers are doing. --- tests/support/server.tcl | 3 +++ tests/test_helper.tcl | 13 +++++++++---- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/tests/support/server.tcl b/tests/support/server.tcl index b20f1ad36..174b05852 100644 --- a/tests/support/server.tcl +++ b/tests/support/server.tcl @@ -53,6 +53,7 @@ proc kill_server config { } # kill server and wait for the process to be totally exited + send_data_packet $::test_server_fd server-killing $pid catch {exec kill $pid} if {$::valgrind} { set max_wait 60000 @@ -231,6 +232,8 @@ proc start_server {options {code undefined}} { set stdout [format "%s/%s" [dict get $config "dir"] "stdout"] set stderr [format "%s/%s" [dict get $config "dir"] "stderr"] + send_data_packet $::test_server_fd "server-spawning" "port $::port" + if {$::valgrind} { set pid [exec valgrind --track-origins=yes --suppressions=src/valgrind.sup --show-reachable=no --show-possibly-lost=no --leak-check=full src/redis-server $config_file > $stdout 2> $stderr &] } elseif ($::stack_logging) { diff --git a/tests/test_helper.tcl b/tests/test_helper.tcl index b266bc56d..fa5579669 100644 --- a/tests/test_helper.tcl +++ b/tests/test_helper.tcl @@ -289,7 +289,7 @@ proc read_from_test_client fd { puts "\[$completed_tests_count/$all_tests_count [colorstr yellow $status]\]: $data ($elapsed seconds)" lappend ::clients_time_history $elapsed $data signal_idle_client $fd - set ::active_clients_task($fd) DONE + set ::active_clients_task($fd) "(DONE) $data" } elseif {$status eq {ok}} { if {!$::quiet} { puts "\[[colorstr green $status]\]: $data" @@ -320,10 +320,16 @@ proc read_from_test_client fd { exit 1 } elseif {$status eq {testing}} { set ::active_clients_task($fd) "(IN PROGRESS) $data" + } elseif {$status eq {server-spawning}} { + set ::active_clients_task($fd) "(SPAWNING SERVER) $data" } elseif {$status eq {server-spawned}} { lappend ::active_servers $data + set ::active_clients_task($fd) "(SPAWNED SERVER) pid:$data" + } elseif {$status eq {server-killing}} { + set ::active_clients_task($fd) "(KILLING SERVER) pid:$data" } elseif {$status eq {server-killed}} { set ::active_servers [lsearch -all -inline -not -exact $::active_servers $data] + set ::active_clients_task($fd) "(KILLED SERVER) pid:$data" } else { if {!$::quiet} { puts "\[$status\]: $data" @@ -333,7 +339,7 @@ proc read_from_test_client fd { proc show_clients_state {} { # The following loop is only useful for debugging tests that may - # enter an infinite loop. Commented out normally. + # enter an infinite loop. foreach x $::active_clients { if {[info exist ::active_clients_task($x)]} { puts "$x => $::active_clients_task($x)" @@ -363,8 +369,6 @@ proc signal_idle_client fd { set ::active_clients \ [lsearch -all -inline -not -exact $::active_clients $fd] - if 0 {show_clients_state} - # New unit to process? if {$::next_test != [llength $::all_tests]} { if {!$::quiet} { @@ -380,6 +384,7 @@ proc signal_idle_client fd { } } else { lappend ::idle_clients $fd + set ::active_clients_task($fd) "SLEEPING, no more units to assign" if {[llength $::active_clients] == 0} { the_end } From 49b80848e72ccb76a09a8a6e8033925562d1a280 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 21 Feb 2020 18:55:56 +0100 Subject: [PATCH 0125/1280] Test engine: detect timeout when checking for Redis startup. --- tests/support/server.tcl | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/tests/support/server.tcl b/tests/support/server.tcl index 174b05852..eec43e485 100644 --- a/tests/support/server.tcl +++ b/tests/support/server.tcl @@ -271,9 +271,19 @@ proc start_server {options {code undefined}} { } # Wait for actual startup + set checkperiod 100; # Milliseconds + set maxiter [expr {120*1000/100}] ; # Wait up to 2 minutes. while {![info exists _pid]} { regexp {PID:\s(\d+)} [exec cat $stdout] _ _pid - after 100 + after $checkperiod + incr maxiter -1 + if {$maxiter == 0} { + start_server_error $config_file "No PID detected in log $stdout" + puts "--- LOG CONTENT ---" + puts [exec cat $stdout] + puts "-------------------" + break + } } # setup properties to be able to initialize a client object From 698be7ee6a06be235748503233d44d93ad202f53 Mon Sep 17 00:00:00 2001 From: chendianqiang Date: Sat, 22 Feb 2020 15:03:01 +0800 Subject: [PATCH 0126/1280] use correct list for moduleUnregisterUsedAPI --- src/module.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/module.c b/src/module.c index aae15d74b..3268f77b2 100644 --- a/src/module.c +++ b/src/module.c @@ -6202,7 +6202,7 @@ int moduleUnregisterUsedAPI(RedisModule *module) { RedisModule *used = ln->value; listNode *ln = listSearchKey(used->usedby,module); if (ln) { - listDelNode(module->using,ln); + listDelNode(used->usedby,ln); count++; } } From 266f2978e4f6271353051beef8e3693314ee1043 Mon Sep 17 00:00:00 2001 From: hwware Date: Sat, 22 Feb 2020 11:38:51 -0500 Subject: [PATCH 0127/1280] add missing file marco --- src/sdsalloc.h | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/sdsalloc.h b/src/sdsalloc.h index 531d41929..c04ff2a0a 100644 --- a/src/sdsalloc.h +++ b/src/sdsalloc.h @@ -36,7 +36,12 @@ * the include of your alternate allocator if needed (not needed in order * to use the default libc allocator). */ +#ifndef __SDS_ALLOC_H__ +#define __SDS_ALLOC_H__ + #include "zmalloc.h" #define s_malloc zmalloc #define s_realloc zrealloc #define s_free zfree + +#endif From 10d0166fea52cbe667b564d72eac803848bd10f1 Mon Sep 17 00:00:00 2001 From: Ariel Date: Sat, 22 Feb 2020 23:49:23 +0200 Subject: [PATCH 0128/1280] fix ThreadSafeContext lock/unlock function names --- src/module.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/module.c b/src/module.c index 705ffec2c..3856fe50e 100644 --- a/src/module.c +++ b/src/module.c @@ -4698,9 +4698,9 @@ int RM_BlockedClientDisconnected(RedisModuleCtx *ctx) { * * To call non-reply APIs, the thread safe context must be prepared with: * - * RedisModule_ThreadSafeCallStart(ctx); + * RedisModule_ThreadSafeContextLock(ctx); * ... make your call here ... - * RedisModule_ThreadSafeCallStop(ctx); + * RedisModule_ThreadSafeContextUnlock(ctx); * * This is not needed when using `RedisModule_Reply*` functions, assuming * that a blocked client was used when the context was created, otherwise From 47cdf342fa60a4ede482cdb1b42d0f22b871eabf Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Sun, 23 Feb 2020 12:46:14 +0200 Subject: [PATCH 0129/1280] Fix latency sensitivity of new defrag test I saw that the new defag test for list was failing in CI recently, so i reduce it's threshold from 12 to 60. besides that, i add / improve the latency test for that other two defrag tests (add a sensitive latency and digest / save checks) and fix bad usage of debug populate (can't overrides existing keys). this was the original intention, which creates higher fragmentation. --- tests/unit/memefficiency.tcl | 40 ++++++++++++++++++++++++++++-------- 1 file changed, 32 insertions(+), 8 deletions(-) diff --git a/tests/unit/memefficiency.tcl b/tests/unit/memefficiency.tcl index ec80c7384..468825a47 100644 --- a/tests/unit/memefficiency.tcl +++ b/tests/unit/memefficiency.tcl @@ -39,6 +39,8 @@ start_server {tags {"memefficiency"}} { start_server {tags {"defrag"}} { if {[string match {*jemalloc*} [s mem_allocator]]} { test "Active defrag" { + r config set save "" ;# prevent bgsave from interfereing with save below + r config set hz 100 r config set activedefrag no r config set active-defrag-threshold-lower 5 r config set active-defrag-cycle-min 65 @@ -46,8 +48,8 @@ start_server {tags {"defrag"}} { r config set active-defrag-ignore-bytes 2mb r config set maxmemory 100mb r config set maxmemory-policy allkeys-lru - r debug populate 700000 asdf 150 - r debug populate 170000 asdf 300 + r debug populate 700000 asdf1 150 + r debug populate 170000 asdf2 300 r ping ;# trigger eviction following the previous population after 120 ;# serverCron only updates the info once in 100ms set frag [s allocator_frag_ratio] @@ -55,6 +57,10 @@ start_server {tags {"defrag"}} { puts "frag $frag" } assert {$frag >= 1.4} + + r config set latency-monitor-threshold 5 + r latency reset + set digest [r debug digest] catch {r config set activedefrag yes} e if {![string match {DISABLED*} $e]} { # Wait for the active defrag to start working (decision once a @@ -78,19 +84,37 @@ start_server {tags {"defrag"}} { # Test the the fragmentation is lower. after 120 ;# serverCron only updates the info once in 100ms set frag [s allocator_frag_ratio] + set max_latency 0 + foreach event [r latency latest] { + lassign $event eventname time latency max + if {$eventname == "active-defrag-cycle"} { + set max_latency $max + } + } if {$::verbose} { puts "frag $frag" + puts "max latency $max_latency" + puts [r latency latest] + puts [r latency history active-defrag-cycle] } assert {$frag < 1.1} + # due to high fragmentation, 100hz, and active-defrag-cycle-max set to 75, + # we expect max latency to be not much higher than 7.5ms but due to rare slowness threshold is set higher + assert {$max_latency <= 60} } else { set _ "" } - } {} + # verify the data isn't corrupted or changed + set newdigest [r debug digest] + assert {$digest eq $newdigest} + r save ;# saving an rdb iterates over all the data / pointers + } {OK} test "Active defrag big keys" { r flushdb r config resetstat r config set save "" ;# prevent bgsave from interfereing with save below + r config set hz 100 r config set activedefrag no r config set active-defrag-max-scan-fields 1000 r config set active-defrag-threshold-lower 5 @@ -200,9 +224,9 @@ start_server {tags {"defrag"}} { puts [r latency history active-defrag-cycle] } assert {$frag < 1.1} - # due to high fragmentation, 10hz, and active-defrag-cycle-max set to 75, - # we expect max latency to be not much higher than 75ms - assert {$max_latency <= 120} + # due to high fragmentation, 100hz, and active-defrag-cycle-max set to 75, + # we expect max latency to be not much higher than 7.5ms but due to rare slowness threshold is set higher + assert {$max_latency <= 60} } # verify the data isn't corrupted or changed set newdigest [r debug digest] @@ -292,8 +316,8 @@ start_server {tags {"defrag"}} { } assert {$frag < 1.1} # due to high fragmentation, 100hz, and active-defrag-cycle-max set to 75, - # we expect max latency to be not much higher than 7.5ms - assert {$max_latency <= 12} + # we expect max latency to be not much higher than 7.5ms but due to rare slowness threshold is set higher + assert {$max_latency <= 60} } # verify the data isn't corrupted or changed set newdigest [r debug digest] From 7d162d74009b731ff10cd02b9c49da613de84e32 Mon Sep 17 00:00:00 2001 From: Guy Benoish Date: Sun, 23 Feb 2020 19:13:09 +0530 Subject: [PATCH 0130/1280] XREADGROUP should propagate XCALIM/SETID in MULTI/EXEC Use built-in alsoPropagate mechanism that wraps commands in MULTI/EXEC before sending them to replica/AOF --- src/t_stream.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/t_stream.c b/src/t_stream.c index 0f0f97a1d..fcfc5ca71 100644 --- a/src/t_stream.c +++ b/src/t_stream.c @@ -848,7 +848,7 @@ void streamPropagateXCLAIM(client *c, robj *key, streamCG *group, robj *groupnam argv[11] = createStringObject("JUSTID",6); argv[12] = createStringObject("LASTID",6); argv[13] = createObjectFromStreamID(&group->last_id); - propagate(server.xclaimCommand,c->db->id,argv,14,PROPAGATE_AOF|PROPAGATE_REPL); + alsoPropagate(server.xclaimCommand,c->db->id,argv,14,PROPAGATE_AOF|PROPAGATE_REPL); decrRefCount(argv[0]); decrRefCount(argv[3]); decrRefCount(argv[4]); @@ -875,7 +875,7 @@ void streamPropagateGroupID(client *c, robj *key, streamCG *group, robj *groupna argv[2] = key; argv[3] = groupname; argv[4] = createObjectFromStreamID(&group->last_id); - propagate(server.xgroupCommand,c->db->id,argv,5,PROPAGATE_AOF|PROPAGATE_REPL); + alsoPropagate(server.xgroupCommand,c->db->id,argv,5,PROPAGATE_AOF|PROPAGATE_REPL); decrRefCount(argv[0]); decrRefCount(argv[1]); decrRefCount(argv[4]); From 462ba8df881a8e913668834dbba95f5ec04c9d1c Mon Sep 17 00:00:00 2001 From: Guy Benoish Date: Wed, 19 Feb 2020 13:24:50 +0530 Subject: [PATCH 0131/1280] Modules: Do not auto-unblock clients if not blocked on keys --- src/module.c | 29 ++++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/src/module.c b/src/module.c index aae15d74b..f37d987e0 100644 --- a/src/module.c +++ b/src/module.c @@ -4290,12 +4290,15 @@ void unblockClientFromModule(client *c) { * We must call moduleUnblockClient in order to free privdata and * RedisModuleBlockedClient. * - * Note that clients implementing threads and working with private data, - * should make sure to stop the threads or protect the private data - * in some other way in the disconnection and timeout callback, because - * here we are going to free the private data associated with the - * blocked client. */ - if (!bc->unblocked) + * Note that we only do that for clients that are blocked on keys, for which + * the contract is that the module should not call RM_UnblockClient under + * normal circumstances. + * Clients implementing threads and working with private data should be + * aware that calling RM_UnblockClient for every blocked client is their + * responsibility, and if they fail to do so memory may leak. Ideally they + * should implement the disconnect and timeout callbacks and call + * RM_UnblockClient, but any other way is also acceptable. */ + if (bc->blocked_on_keys && !bc->unblocked) moduleUnblockClient(c); bc->client = NULL; @@ -4409,6 +4412,10 @@ int moduleTryServeClientBlockedOnKey(client *c, robj *key) { * * free_privdata: called in order to free the private data that is passed * by RedisModule_UnblockClient() call. + * + * Note: RedisModule_UnblockClient should be called for every blocked client, + * even if client was killed, timed-out or disconnected. Failing to do so + * will result in memory leaks. */ RedisModuleBlockedClient *RM_BlockClient(RedisModuleCtx *ctx, RedisModuleCmdFunc reply_callback, RedisModuleCmdFunc timeout_callback, void (*free_privdata)(RedisModuleCtx*,void*), long long timeout_ms) { return moduleBlockClient(ctx,reply_callback,timeout_callback,free_privdata,timeout_ms, NULL,0,NULL); @@ -4463,7 +4470,15 @@ RedisModuleBlockedClient *RM_BlockClient(RedisModuleCtx *ctx, RedisModuleCmdFunc * freed using the free_privdata callback provided by the user. * * However the reply callback will be able to access the argument vector of - * the command, so the private data is often not needed. */ + * the command, so the private data is often not needed. + * + * Note: Under normal circumstances RedisModule_UnblockClient should not be + * called for clients that are blocked on keys (Either the key will + * become ready or a timeout will occur). If for some reason you do want + * to call RedisModule_UnblockClient it is possible: Client will be + * handled as if it were timed-out (You must implement the timeout + * callback in that case). + */ RedisModuleBlockedClient *RM_BlockClientOnKeys(RedisModuleCtx *ctx, RedisModuleCmdFunc reply_callback, RedisModuleCmdFunc timeout_callback, void (*free_privdata)(RedisModuleCtx*,void*), long long timeout_ms, RedisModuleString **keys, int numkeys, void *privdata) { return moduleBlockClient(ctx,reply_callback,timeout_callback,free_privdata,timeout_ms, keys,numkeys,privdata); } From 8e4b65874cd1ef8434382723fe26955d6d6a89d3 Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Thu, 20 Feb 2020 17:56:52 +0200 Subject: [PATCH 0132/1280] fix race in module api test for fork in some cases we were trying to kill the fork before it got created --- tests/modules/fork.c | 2 +- tests/unit/moduleapi/fork.tcl | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/modules/fork.c b/tests/modules/fork.c index 1a139ef1b..0443d9ef0 100644 --- a/tests/modules/fork.c +++ b/tests/modules/fork.c @@ -42,7 +42,7 @@ int fork_create(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) /* child */ RedisModule_Log(ctx, "notice", "fork child started"); - usleep(200000); + usleep(500000); RedisModule_Log(ctx, "notice", "fork child exiting"); RedisModule_ExitFromChild(code_to_exit_with); /* unreachable */ diff --git a/tests/unit/moduleapi/fork.tcl b/tests/unit/moduleapi/fork.tcl index f7d7e47d5..8535a3382 100644 --- a/tests/unit/moduleapi/fork.tcl +++ b/tests/unit/moduleapi/fork.tcl @@ -20,9 +20,8 @@ start_server {tags {"modules"}} { test {Module fork kill} { r fork.create 3 - after 20 + after 250 r fork.kill - after 100 assert {[count_log_message "fork child started"] eq "2"} assert {[count_log_message "Received SIGUSR1 in child"] eq "1"} From ab71e1020c5493ff376303beb8760550fd087f7e Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 24 Feb 2020 10:46:23 +0100 Subject: [PATCH 0133/1280] Test engine: experimental change to avoid busy port problems. --- tests/support/server.tcl | 149 ++++++++++++++++++++++++--------------- 1 file changed, 92 insertions(+), 57 deletions(-) diff --git a/tests/support/server.tcl b/tests/support/server.tcl index eec43e485..d086366dc 100644 --- a/tests/support/server.tcl +++ b/tests/support/server.tcl @@ -141,6 +141,18 @@ proc tags {tags code} { uplevel 1 $code set ::tags [lrange $::tags 0 end-[llength $tags]] } + +# Write the configuration in the dictionary 'config' in the specified +# file name. +proc create_server_config_file {filename config} { + set fp [open $filename w+] + foreach directive [dict keys $config] { + puts -nonewline $fp "$directive " + puts $fp [dict get $config $directive] + } + close $fp +} + proc start_server {options {code undefined}} { # If we are running against an external server, we just push the # host/port pair in the stack the first time @@ -222,68 +234,91 @@ proc start_server {options {code undefined}} { # write new configuration to temporary file set config_file [tmpfile redis.conf] - set fp [open $config_file w+] - foreach directive [dict keys $config] { - puts -nonewline $fp "$directive " - puts $fp [dict get $config $directive] - } - close $fp + create_server_config_file $config_file $config set stdout [format "%s/%s" [dict get $config "dir"] "stdout"] set stderr [format "%s/%s" [dict get $config "dir"] "stderr"] - send_data_packet $::test_server_fd "server-spawning" "port $::port" - - if {$::valgrind} { - set pid [exec valgrind --track-origins=yes --suppressions=src/valgrind.sup --show-reachable=no --show-possibly-lost=no --leak-check=full src/redis-server $config_file > $stdout 2> $stderr &] - } elseif ($::stack_logging) { - set pid [exec /usr/bin/env MallocStackLogging=1 MallocLogFile=/tmp/malloc_log.txt src/redis-server $config_file > $stdout 2> $stderr &] - } else { - set pid [exec src/redis-server $config_file > $stdout 2> $stderr &] - } - - # Tell the test server about this new instance. - send_data_packet $::test_server_fd server-spawned $pid - - # check that the server actually started - # ugly but tries to be as fast as possible... - if {$::valgrind} {set retrynum 1000} else {set retrynum 100} - - if {$::verbose} { - puts -nonewline "=== ($tags) Starting server ${::host}:${::port} " - } - - if {$code ne "undefined"} { - set serverisup [server_is_up $::host $::port $retrynum] - } else { - set serverisup 1 - } - - if {$::verbose} { - puts "" - } - - if {!$serverisup} { - set err {} - append err [exec cat $stdout] "\n" [exec cat $stderr] - start_server_error $config_file $err - return - } - - # Wait for actual startup - set checkperiod 100; # Milliseconds - set maxiter [expr {120*1000/100}] ; # Wait up to 2 minutes. - while {![info exists _pid]} { - regexp {PID:\s(\d+)} [exec cat $stdout] _ _pid - after $checkperiod - incr maxiter -1 - if {$maxiter == 0} { - start_server_error $config_file "No PID detected in log $stdout" - puts "--- LOG CONTENT ---" - puts [exec cat $stdout] - puts "-------------------" - break + # We need a loop here to retry with different ports. + set server_started 0 + while {$server_started == 0} { + if {$::verbose} { + puts -nonewline "=== ($tags) Starting server ${::host}:${::port} " } + + send_data_packet $::test_server_fd "server-spawning" "port $::port" + + if {$::valgrind} { + set pid [exec valgrind --track-origins=yes --suppressions=src/valgrind.sup --show-reachable=no --show-possibly-lost=no --leak-check=full src/redis-server $config_file > $stdout 2> $stderr &] + } elseif ($::stack_logging) { + set pid [exec /usr/bin/env MallocStackLogging=1 MallocLogFile=/tmp/malloc_log.txt src/redis-server $config_file > $stdout 2> $stderr &] + } else { + set pid [exec src/redis-server $config_file > $stdout 2> $stderr &] + } + + # Tell the test server about this new instance. + send_data_packet $::test_server_fd server-spawned $pid + + # check that the server actually started + # ugly but tries to be as fast as possible... + if {$::valgrind} {set retrynum 1000} else {set retrynum 100} + + # Wait for actual startup + set checkperiod 100; # Milliseconds + set maxiter [expr {120*1000/100}] ; # Wait up to 2 minutes. + set port_busy 0 + while {![info exists _pid]} { + regexp {PID:\s(\d+)} [exec cat $stdout] _ _pid + after $checkperiod + incr maxiter -1 + if {$maxiter == 0} { + start_server_error $config_file "No PID detected in log $stdout" + puts "--- LOG CONTENT ---" + puts [exec cat $stdout] + puts "-------------------" + break + } + + # Check if the port is actually busy and the server failed + # for this reason. + if {[regexp {Could not create server TCP} [exec cat $stdout]]} { + set port_busy 1 + break + } + } + + # Sometimes we have to try a different port, even if we checked + # for availability. Other test clients may grab the port before we + # are able to do it for example. + if {$port_busy} { + puts "Port $::port was already busy, trying another port..." + set ::port [find_available_port [expr {$::port+1}]] + if {$::tls} { + dict set config "tls-port" $::port + } else { + dict set config port $::port + } + create_server_config_file $config_file $config + continue; # Try again + } + + if {$code ne "undefined"} { + set serverisup [server_is_up $::host $::port $retrynum] + } else { + set serverisup 1 + } + + if {$::verbose} { + puts "" + } + + if {!$serverisup} { + set err {} + append err [exec cat $stdout] "\n" [exec cat $stderr] + start_server_error $config_file $err + return + } + set server_started 1 } # setup properties to be able to initialize a client object From b5e379004f7880283987917b0c9bc0984b247b3e Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 21 Feb 2020 16:39:42 +0100 Subject: [PATCH 0134/1280] Tracking: optin/out implemented. --- src/networking.c | 65 ++++++++++++++++++++++++++++++++++++++++++------ src/server.h | 6 ++++- src/tracking.c | 27 ++++++++++++++------ 3 files changed, 82 insertions(+), 16 deletions(-) diff --git a/src/networking.c b/src/networking.c index 5b1229fde..4c394af70 100644 --- a/src/networking.c +++ b/src/networking.c @@ -1365,6 +1365,12 @@ void resetClient(client *c) { if (!(c->flags & CLIENT_MULTI) && prevcmd != askingCommand) c->flags &= ~CLIENT_ASKING; + /* We do the same for the CACHING command as well. It also affects + * the next command or transaction executed, in a way very similar + * to ASKING. */ + if (!(c->flags & CLIENT_MULTI) && prevcmd != clientCommand) + c->flags &= ~CLIENT_TRACKING_CACHING; + /* Remove the CLIENT_REPLY_SKIP flag if any so that the reply * to the next command will be sent, but set the flag if the command * we just processed was "CLIENT REPLY SKIP". */ @@ -2044,7 +2050,7 @@ void clientCommand(client *c) { "REPLY (on|off|skip) -- Control the replies sent to the current connection.", "SETNAME -- Assign the name to the current connection.", "UNBLOCK [TIMEOUT|ERROR] -- Unblock the specified blocked client.", -"TRACKING (on|off) [REDIRECT ] [BCAST] [PREFIX first] [PREFIX second] ... -- Enable client keys tracking for client side caching.", +"TRACKING (on|off) [REDIRECT ] [BCAST] [PREFIX first] [PREFIX second] [OPTIN] [OPTOUT]... -- Enable client keys tracking for client side caching.", "GETREDIR -- Return the client ID we are redirecting to when tracking is enabled.", NULL }; @@ -2221,9 +2227,9 @@ NULL addReply(c,shared.ok); } else if (!strcasecmp(c->argv[1]->ptr,"tracking") && c->argc >= 3) { /* CLIENT TRACKING (on|off) [REDIRECT ] [BCAST] [PREFIX first] - * [PREFIX second] ... */ + * [PREFIX second] [OPTIN] [OPTOUT] ... */ long long redir = 0; - int bcast = 0; + uint64_t options = 0; robj **prefix = NULL; size_t numprefix = 0; @@ -2256,7 +2262,11 @@ NULL return; } } else if (!strcasecmp(c->argv[j]->ptr,"bcast")) { - bcast = 1; + options |= CLIENT_TRACKING_BCAST; + } else if (!strcasecmp(c->argv[j]->ptr,"optin")) { + options |= CLIENT_TRACKING_OPTIN; + } else if (!strcasecmp(c->argv[j]->ptr,"optout")) { + options |= CLIENT_TRACKING_OPTOUT; } else if (!strcasecmp(c->argv[j]->ptr,"prefix") && moreargs) { j++; prefix = zrealloc(prefix,sizeof(robj*)*(numprefix+1)); @@ -2272,7 +2282,7 @@ NULL if (!strcasecmp(c->argv[2]->ptr,"on")) { /* Before enabling tracking, make sure options are compatible * among each other and with the current state of the client. */ - if (!bcast && numprefix) { + if (!(options & CLIENT_TRACKING_BCAST) && numprefix) { addReplyError(c, "PREFIX option requires BCAST mode to be enabled"); zfree(prefix); @@ -2281,7 +2291,8 @@ NULL if (c->flags & CLIENT_TRACKING) { int oldbcast = !!(c->flags & CLIENT_TRACKING_BCAST); - if (oldbcast != bcast) { + int newbcast = !!(options & CLIENT_TRACKING_BCAST); + if (oldbcast != newbcast) { addReplyError(c, "You can't switch BCAST mode on/off before disabling " "tracking for this client, and then re-enabling it with " @@ -2290,7 +2301,17 @@ NULL return; } } - enableTracking(c,redir,bcast,prefix,numprefix); + + if (options & CLIENT_TRACKING_BCAST && + options & (CLIENT_TRACKING_OPTIN|CLIENT_TRACKING_OPTOUT)) + { + addReplyError(c, + "OPTIN and OPTOUT are not compatible with BCAST"); + zfree(prefix); + return; + } + + enableTracking(c,redir,options,prefix,numprefix); } else if (!strcasecmp(c->argv[2]->ptr,"off")) { disableTracking(c); } else { @@ -2300,6 +2321,36 @@ NULL } zfree(prefix); addReply(c,shared.ok); + } else if (!strcasecmp(c->argv[1]->ptr,"caching") && c->argc >= 3) { + if (!(c->flags & CLIENT_TRACKING)) { + addReplyError(c,"CLIENT CACHING can be called only when the " + "client is in tracking mode with OPTIN or " + "OPTOUT mode enabled"); + return; + } + + char *opt = c->argv[2]->ptr; + if (!strcasecmp(opt,"yes")) { + if (c->flags & CLIENT_TRACKING_OPTIN) { + c->flags |= CLIENT_TRACKING_CACHING; + } else { + addReplyError(c,"CLIENT CACHING YES is only valid when tracking is enabled in OPTIN mode."); + return; + } + } else if (!strcasecmp(opt,"no")) { + if (c->flags & CLIENT_TRACKING_OPTOUT) { + c->flags |= CLIENT_TRACKING_CACHING; + } else { + addReplyError(c,"CLIENT CACHING NO is only valid when tracking is enabled in OPTOUT mode."); + return; + } + } else { + addReply(c,shared.syntaxerr); + return; + } + + /* Common reply for when we succeeded. */ + addReply(c,shared.ok); } else if (!strcasecmp(c->argv[1]->ptr,"getredir") && c->argc == 2) { /* CLIENT GETREDIR */ if (c->flags & CLIENT_TRACKING) { diff --git a/src/server.h b/src/server.h index bb40ca4e1..87c293c26 100644 --- a/src/server.h +++ b/src/server.h @@ -248,6 +248,10 @@ typedef long long ustime_t; /* microsecond time type. */ perform client side caching. */ #define CLIENT_TRACKING_BROKEN_REDIR (1ULL<<32) /* Target client is invalid. */ #define CLIENT_TRACKING_BCAST (1ULL<<33) /* Tracking in BCAST mode. */ +#define CLIENT_TRACKING_OPTIN (1ULL<<34) /* Tracking in opt-in mode. */ +#define CLIENT_TRACKING_OPTOUT (1ULL<<35) /* Tracking in opt-out mode. */ +#define CLIENT_TRACKING_CACHING (1ULL<<36) /* CACHING yes/no was given, + depending on optin/optout mode. */ /* Client block type (btype field in client structure) * if CLIENT_BLOCKED flag is set. */ @@ -1651,7 +1655,7 @@ void addReplyStatusFormat(client *c, const char *fmt, ...); #endif /* Client side caching (tracking mode) */ -void enableTracking(client *c, uint64_t redirect_to, int bcast, robj **prefix, size_t numprefix); +void enableTracking(client *c, uint64_t redirect_to, uint64_t options, robj **prefix, size_t numprefix); void disableTracking(client *c); void trackingRememberKeys(client *c); void trackingInvalidateKey(robj *keyobj); diff --git a/src/tracking.c b/src/tracking.c index 619148f2f..45f83103a 100644 --- a/src/tracking.c +++ b/src/tracking.c @@ -93,7 +93,8 @@ void disableTracking(client *c) { if (c->flags & CLIENT_TRACKING) { server.tracking_clients--; c->flags &= ~(CLIENT_TRACKING|CLIENT_TRACKING_BROKEN_REDIR| - CLIENT_TRACKING_BCAST); + CLIENT_TRACKING_BCAST|CLIENT_TRACKING_OPTIN| + CLIENT_TRACKING_OPTOUT); } } @@ -124,10 +125,11 @@ void enableBcastTrackingForPrefix(client *c, char *prefix, size_t plen) { * eventually get freed, we'll send a message to the original client to * inform it of the condition. Multiple clients can redirect the invalidation * messages to the same client ID. */ -void enableTracking(client *c, uint64_t redirect_to, int bcast, robj **prefix, size_t numprefix) { +void enableTracking(client *c, uint64_t redirect_to, uint64_t options, robj **prefix, size_t numprefix) { if (!(c->flags & CLIENT_TRACKING)) server.tracking_clients++; c->flags |= CLIENT_TRACKING; - c->flags &= ~(CLIENT_TRACKING_BROKEN_REDIR|CLIENT_TRACKING_BCAST); + c->flags &= ~(CLIENT_TRACKING_BROKEN_REDIR|CLIENT_TRACKING_BCAST| + CLIENT_TRACKING_OPTIN|CLIENT_TRACKING_OPTOUT); c->client_tracking_redirection = redirect_to; if (TrackingTable == NULL) { TrackingTable = raxNew(); @@ -135,7 +137,7 @@ void enableTracking(client *c, uint64_t redirect_to, int bcast, robj **prefix, s TrackingChannelName = createStringObject("__redis__:invalidate",20); } - if (bcast) { + if (options & CLIENT_TRACKING_BCAST) { c->flags |= CLIENT_TRACKING_BCAST; if (numprefix == 0) enableBcastTrackingForPrefix(c,"",0); for (size_t j = 0; j < numprefix; j++) { @@ -143,14 +145,23 @@ void enableTracking(client *c, uint64_t redirect_to, int bcast, robj **prefix, s enableBcastTrackingForPrefix(c,sdsprefix,sdslen(sdsprefix)); } } + c->flags |= options & (CLIENT_TRACKING_OPTIN|CLIENT_TRACKING_OPTOUT); } /* This function is called after the execution of a readonly command in the - * case the client 'c' has keys tracking enabled. It will populate the - * tracking invalidation table according to the keys the user fetched, so that - * Redis will know what are the clients that should receive an invalidation - * message with certain groups of keys are modified. */ + * case the client 'c' has keys tracking enabled and the tracking is not + * in BCAST mode. It will populate the tracking invalidation table according + * to the keys the user fetched, so that Redis will know what are the clients + * that should receive an invalidation message with certain groups of keys + * are modified. */ void trackingRememberKeys(client *c) { + /* Return if we are in optin/out mode and the right CACHING command + * was/wasn't given in order to modify the default behavior. */ + uint64_t optin = c->flags & CLIENT_TRACKING_OPTIN; + uint64_t optout = c->flags & CLIENT_TRACKING_OPTOUT; + uint64_t caching_given = c->flags & CLIENT_TRACKING_CACHING; + if ((optin && !caching_given) || (optout && caching_given)) return; + int numkeys; int *keys = getKeysFromCommand(c->cmd,c->argv,c->argc,&numkeys); if (keys == NULL) return; From 6b55447c1d508bc18a41e151955c49be02df4724 Mon Sep 17 00:00:00 2001 From: Hengjian Tang Date: Tue, 25 Feb 2020 15:55:28 +0800 Subject: [PATCH 0135/1280] modify the read buf size according to the write buf size PROTO_IOBUF_LEN defined before --- src/replication.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/replication.c b/src/replication.c index 4843f97d5..c497051c8 100644 --- a/src/replication.c +++ b/src/replication.c @@ -1352,7 +1352,7 @@ void disklessLoadRestoreBackups(redisDb *backup, int restore, int empty_db_flags /* Asynchronously read the SYNC payload we receive from a master */ #define REPL_MAX_WRITTEN_BEFORE_FSYNC (1024*1024*8) /* 8 MB */ void readSyncBulkPayload(connection *conn) { - char buf[4096]; + char buf[PROTO_IOBUF_LEN]; ssize_t nread, readlen, nwritten; int use_diskless_load = useDisklessLoad(); redisDb *diskless_load_backup = NULL; From 7009978e25789ac39e4ab5e6dfda20cf00d230a9 Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Tue, 25 Feb 2020 13:01:52 +0200 Subject: [PATCH 0136/1280] fix github actions failing latency test for active defrag seems that github actions are slow, using just one client to reduce false positives. also adding verbose, testing only on latest ubuntu, and building on older one. when doing that, i can reduce the test threshold back to something saner --- .github/workflows/ci.yml | 23 ++++++++++++----------- tests/unit/memefficiency.tcl | 6 +++--- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 847abcf02..559ae61d8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3,11 +3,8 @@ name: CI on: [push, pull_request] jobs: - build-ubuntu: - strategy: - matrix: - platform: [ubuntu-latest, ubuntu-16.04] - runs-on: ${{ matrix.platform }} + test-ubuntu-latest: + runs-on: ubuntu-latest steps: - uses: actions/checkout@v1 - name: make @@ -15,13 +12,17 @@ jobs: - name: test run: | sudo apt-get install tcl8.5 - make test + ./runtest --clients 1 --verbose - build-macos-latest: - strategy: - matrix: - platform: [macos-latest, macOS-10.14] - runs-on: ${{ matrix.platform }} + test-ubuntu-old: + runs-on: ubuntu-16.04 + steps: + - uses: actions/checkout@v1 + - name: make + run: make + + build-macos-latest: + runs-on: macos-latest steps: - uses: actions/checkout@v1 - name: make diff --git a/tests/unit/memefficiency.tcl b/tests/unit/memefficiency.tcl index 468825a47..c899103fd 100644 --- a/tests/unit/memefficiency.tcl +++ b/tests/unit/memefficiency.tcl @@ -100,7 +100,7 @@ start_server {tags {"defrag"}} { assert {$frag < 1.1} # due to high fragmentation, 100hz, and active-defrag-cycle-max set to 75, # we expect max latency to be not much higher than 7.5ms but due to rare slowness threshold is set higher - assert {$max_latency <= 60} + assert {$max_latency <= 30} } else { set _ "" } @@ -226,7 +226,7 @@ start_server {tags {"defrag"}} { assert {$frag < 1.1} # due to high fragmentation, 100hz, and active-defrag-cycle-max set to 75, # we expect max latency to be not much higher than 7.5ms but due to rare slowness threshold is set higher - assert {$max_latency <= 60} + assert {$max_latency <= 30} } # verify the data isn't corrupted or changed set newdigest [r debug digest] @@ -317,7 +317,7 @@ start_server {tags {"defrag"}} { assert {$frag < 1.1} # due to high fragmentation, 100hz, and active-defrag-cycle-max set to 75, # we expect max latency to be not much higher than 7.5ms but due to rare slowness threshold is set higher - assert {$max_latency <= 60} + assert {$max_latency <= 30} } # verify the data isn't corrupted or changed set newdigest [r debug digest] From 374f0f4a16521747b983a817902572aaede401d0 Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Wed, 26 Feb 2020 08:12:07 +0200 Subject: [PATCH 0137/1280] fix github actions failing latency test for active defrag - part 2 it seems that running two clients at a time is ok too, resuces action time from 20 minutes to 10. we'll use this for now, and if one day it won't be enough we'll have to run just the sensitive tests one by one separately from the others. this commit also fixes an issue with the defrag test that appears to be very rare. --- .github/workflows/ci.yml | 4 ++-- tests/unit/memefficiency.tcl | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 559ae61d8..cc4991606 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,9 +12,9 @@ jobs: - name: test run: | sudo apt-get install tcl8.5 - ./runtest --clients 1 --verbose + ./runtest --clients 2 --verbose - test-ubuntu-old: + build-ubuntu-old: runs-on: ubuntu-16.04 steps: - uses: actions/checkout@v1 diff --git a/tests/unit/memefficiency.tcl b/tests/unit/memefficiency.tcl index c899103fd..06b0e07d7 100644 --- a/tests/unit/memefficiency.tcl +++ b/tests/unit/memefficiency.tcl @@ -60,6 +60,7 @@ start_server {tags {"defrag"}} { r config set latency-monitor-threshold 5 r latency reset + r config set maxmemory 110mb ;# prevent further eviction (not to fail the digest test) set digest [r debug digest] catch {r config set activedefrag yes} e if {![string match {DISABLED*} $e]} { @@ -166,7 +167,7 @@ start_server {tags {"defrag"}} { for {set j 0} {$j < 500000} {incr j} { $rd read ; # Discard replies } - assert {[r dbsize] == 500010} + assert_equal [r dbsize] 500010 # create some fragmentation for {set j 0} {$j < 500000} {incr j 2} { @@ -175,7 +176,7 @@ start_server {tags {"defrag"}} { for {set j 0} {$j < 500000} {incr j 2} { $rd read ; # Discard replies } - assert {[r dbsize] == 250010} + assert_equal [r dbsize] 250010 # start defrag after 120 ;# serverCron only updates the info once in 100ms From e074a2ea875da366cb451fca680b1a69cf1102de Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Sun, 23 Feb 2020 16:51:27 +0200 Subject: [PATCH 0138/1280] change CI to build and run the module api tests --- .github/workflows/ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cc4991606..3a81d1a08 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,6 +13,8 @@ jobs: run: | sudo apt-get install tcl8.5 ./runtest --clients 2 --verbose + - name: module api test + run: ./runtest-moduleapi --clients 2 --verbose build-ubuntu-old: runs-on: ubuntu-16.04 From 39cc45d5e5c8a27ca218fca443019447e09d7660 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 27 Feb 2020 17:41:48 +0100 Subject: [PATCH 0139/1280] Improve aeDeleteEventLoop() top comment grammar. --- src/ae.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/ae.c b/src/ae.c index 248096e1f..d2248fe5c 100644 --- a/src/ae.c +++ b/src/ae.c @@ -135,7 +135,8 @@ void aeDeleteEventLoop(aeEventLoop *eventLoop) { aeApiFree(eventLoop); zfree(eventLoop->events); zfree(eventLoop->fired); - /* Free time event. */ + + /* Free the time events list. */ aeTimeEvent *next_te, *te = eventLoop->timeEventHead; while (te) { next_te = te->next; From adaa89c8a496c26ed2d5c1e4e921f28634531c8e Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 27 Feb 2020 17:45:48 +0100 Subject: [PATCH 0140/1280] Remove useless comment from enumConfigSet(). --- src/config.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/config.c b/src/config.c index 5841ae7a0..8d069f8db 100644 --- a/src/config.c +++ b/src/config.c @@ -1674,7 +1674,6 @@ static int enumConfigSet(typeData data, sds value, int update, char **err) { enumerr[sdslen(enumerr) - 2] = '\0'; strncpy(loadbuf, enumerr, LOADBUF_SIZE); - /* strncpy does not if null terminate if source string length is >= destination buffer. */ loadbuf[LOADBUF_SIZE - 1] = '\0'; sdsfree(enumerr); From e45e07a2688b4d6b0f9b8e6049ade63e8e36bf7a Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 27 Feb 2020 17:47:50 +0100 Subject: [PATCH 0141/1280] Fix SDS misuse in enumConfigSet(). Related to #6778. --- src/config.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/config.c b/src/config.c index 8d069f8db..aeb2fea7e 100644 --- a/src/config.c +++ b/src/config.c @@ -1666,12 +1666,12 @@ static int enumConfigSet(typeData data, sds value, int update, char **err) { sds enumerr = sdsnew("argument must be one of the following: "); configEnum *enumNode = data.enumd.enum_value; while(enumNode->name != NULL) { - enumerr = sdscatlen(enumerr, enumNode->name, strlen(enumNode->name)); + enumerr = sdscatlen(enumerr, enumNode->name, + strlen(enumNode->name)); enumerr = sdscatlen(enumerr, ", ", 2); enumNode++; } - - enumerr[sdslen(enumerr) - 2] = '\0'; + sdsrange(enumerr,0,-3); /* Remove final ", ". */ strncpy(loadbuf, enumerr, LOADBUF_SIZE); loadbuf[LOADBUF_SIZE - 1] = '\0'; From efd2cf9b7898e9f08e8097015a910531937d5305 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 27 Feb 2020 18:21:12 +0100 Subject: [PATCH 0142/1280] Show Redis version when not understanding a config directive. This makes simpler to give people help when posting such kind of errors in the mailing list or other help forums, because sometimes the directive looks well spelled, but the version of Redis they are using is not able to support it. --- src/config.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/config.c b/src/config.c index aeb2fea7e..fd04b7c87 100644 --- a/src/config.c +++ b/src/config.c @@ -518,7 +518,8 @@ void loadServerConfigFromString(char *config) { return; loaderr: - fprintf(stderr, "\n*** FATAL CONFIG FILE ERROR ***\n"); + fprintf(stderr, "\n*** FATAL CONFIG FILE ERROR (Redis %s) ***\n", + REDIS_VERSION); fprintf(stderr, "Reading the configuration file, at line %d\n", linenum); fprintf(stderr, ">>> '%s'\n", lines[i]); fprintf(stderr, "%s\n", err); From 09cd0bf4a9c1cad729cfb99966202e4014f497a9 Mon Sep 17 00:00:00 2001 From: Itamar Haber Date: Fri, 28 Feb 2020 13:35:10 +0200 Subject: [PATCH 0143/1280] Adds `BIN_PATH` to create-cluster Allows for setting the binaries path if used outside the upstream repo. Also documents `call` in usage clause (TODO: port to `redis-cli --cluster call` or just deprecate it). --- utils/create-cluster/create-cluster | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/utils/create-cluster/create-cluster b/utils/create-cluster/create-cluster index 4aef0816a..5b8bfe250 100755 --- a/utils/create-cluster/create-cluster +++ b/utils/create-cluster/create-cluster @@ -1,6 +1,7 @@ #!/bin/bash # Settings +BIN_PATH="../../" CLUSTER_HOST=127.0.0.1 PORT=30000 TIMEOUT=2000 @@ -25,7 +26,7 @@ then while [ $((PORT < ENDPORT)) != "0" ]; do PORT=$((PORT+1)) echo "Starting $PORT" - ../../src/redis-server --port $PORT --protected-mode $PROTECTED_MODE --cluster-enabled yes --cluster-config-file nodes-${PORT}.conf --cluster-node-timeout $TIMEOUT --appendonly yes --appendfilename appendonly-${PORT}.aof --dbfilename dump-${PORT}.rdb --logfile ${PORT}.log --daemonize yes ${ADDITIONAL_OPTIONS} + $BIN_PATH/redis-server --port $PORT --protected-mode $PROTECTED_MODE --cluster-enabled yes --cluster-config-file nodes-${PORT}.conf --cluster-node-timeout $TIMEOUT --appendonly yes --appendfilename appendonly-${PORT}.aof --dbfilename dump-${PORT}.rdb --logfile ${PORT}.log --daemonize yes ${ADDITIONAL_OPTIONS} done exit 0 fi @@ -37,7 +38,7 @@ then PORT=$((PORT+1)) HOSTS="$HOSTS $CLUSTER_HOST:$PORT" done - ../../src/redis-cli --cluster create $HOSTS --cluster-replicas $REPLICAS + $BIN_PATH/redis-cli --cluster create $HOSTS --cluster-replicas $REPLICAS exit 0 fi @@ -46,7 +47,7 @@ then while [ $((PORT < ENDPORT)) != "0" ]; do PORT=$((PORT+1)) echo "Stopping $PORT" - ../../src/redis-cli -p $PORT shutdown nosave + $BIN_PATH/redis-cli -p $PORT shutdown nosave done exit 0 fi @@ -57,7 +58,7 @@ then while [ 1 ]; do clear date - ../../src/redis-cli -p $PORT cluster nodes | head -30 + $BIN_PATH/redis-cli -p $PORT cluster nodes | head -30 sleep 1 done exit 0 @@ -81,7 +82,7 @@ if [ "$1" == "call" ] then while [ $((PORT < ENDPORT)) != "0" ]; do PORT=$((PORT+1)) - ../../src/redis-cli -p $PORT $2 $3 $4 $5 $6 $7 $8 $9 + $BIN_PATH/redis-cli -p $PORT $2 $3 $4 $5 $6 $7 $8 $9 done exit 0 fi @@ -101,7 +102,7 @@ then exit 0 fi -echo "Usage: $0 [start|create|stop|watch|tail|clean]" +echo "Usage: $0 [start|create|stop|watch|tail|clean|call]" echo "start -- Launch Redis Cluster instances." echo "create -- Create a cluster using redis-cli --cluster create." echo "stop -- Stop Redis Cluster instances." @@ -110,3 +111,4 @@ echo "tail -- Run tail -f of instance at base port + ID." echo "tailall -- Run tail -f for all the log files at once." echo "clean -- Remove all instances data, logs, configs." echo "clean-logs -- Remove just instances logs." +echo "call -- Call a command (up to 7 arguments) on all nodes. " From 5a4bb229920faa86cba2a655c170b02c159ade69 Mon Sep 17 00:00:00 2001 From: Itamar Haber Date: Fri, 28 Feb 2020 13:36:50 +0200 Subject: [PATCH 0144/1280] Update create-cluster --- utils/create-cluster/create-cluster | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/create-cluster/create-cluster b/utils/create-cluster/create-cluster index 5b8bfe250..4db0a6619 100755 --- a/utils/create-cluster/create-cluster +++ b/utils/create-cluster/create-cluster @@ -111,4 +111,4 @@ echo "tail -- Run tail -f of instance at base port + ID." echo "tailall -- Run tail -f for all the log files at once." echo "clean -- Remove all instances data, logs, configs." echo "clean-logs -- Remove just instances logs." -echo "call -- Call a command (up to 7 arguments) on all nodes. " +echo "call -- Call a command (up to 7 arguments) on all nodes." From 720a07805ec3fbf6fb30a4ebeee3e6a61402eb7e Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 28 Feb 2020 18:06:30 +0100 Subject: [PATCH 0145/1280] Modules: more details in RM_Scan API top comment. --- src/module.c | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/src/module.c b/src/module.c index b821f0c31..47371fb32 100644 --- a/src/module.c +++ b/src/module.c @@ -6557,9 +6557,21 @@ void RM_ScanCursorDestroy(RedisModuleScanCursor *cursor) { * possibly setting errno if the call failed. * It is also possible to restart and existing cursor using RM_CursorRestart. * - * NOTE: You must avoid doing any database changes from within the callback, you should avoid any - * RedisModule_OpenKey or RedisModule_Call, if you need to do these, you need to keep the key name - * and do any work you need to do after the call to Scan returns. */ + * IMPORTANT: This API is very similar to the Redis SCAN command from the + * point of view of the guarantees it provides. This means that the API + * may report duplicated keys, but guarantees to report at least one time + * every key that was there from the start to the end of the scanning process. + * + * NOTE: If you do database changes within the callback, you should be aware + * that the internal state of the database may change. For instance it is safe + * to delete or modify the current key, but may not be safe to delete any + * other key. + * Moreover playing with the Redis keyspace while iterating may have the + * effect of returning more duplicates. A safe pattern is to store the keys + * names you want to modify elsewhere, and perform the actions on the keys + * later when the iteration is complete. Howerver this can cost a lot of + * memory, so it may make sense to just operate on the current key when + * possible during the iteration, given that this is safe. */ int RM_Scan(RedisModuleCtx *ctx, RedisModuleScanCursor *cursor, RedisModuleScanCB fn, void *privdata) { if (cursor->done) { errno = ENOENT; @@ -6641,9 +6653,13 @@ static void moduleScanKeyCallback(void *privdata, const dictEntry *de) { * possibly setting errno if the call failed. * It is also possible to restart and existing cursor using RM_CursorRestart. * - * NOTE: You must avoid doing any database changes from within the callback, you should avoid any - * RedisModule_OpenKey or RedisModule_Call, if you need to do these, you need to keep the field name - * and do any work you need to do after the call to Scan returns. */ + * NOTE: Certain operations are unsafe while iterating the object. For instance + * while the API guarantees to return at least one time all the elements that + * are present in the data structure consistently from the start to the end + * of the iteration (see HSCAN and similar commands documentation), the more + * you play with the elements, the more duplicates you may get. In general + * deleting the current element of the data structure is safe, while removing + * the key you are iterating is not safe. */ int RM_ScanKey(RedisModuleKey *key, RedisModuleScanCursor *cursor, RedisModuleScanKeyCB fn, void *privdata) { if (key == NULL || key->value == NULL) { errno = EINVAL; From 7e8637d87eb2ba34304748ab6ffc9b2f195d4a08 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 28 Feb 2020 18:09:46 +0100 Subject: [PATCH 0146/1280] Modules: reformat RM_Scan() top comment a bit. --- src/module.c | 33 +++++++++++++++++++++------------ 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/src/module.c b/src/module.c index 47371fb32..bbd54082c 100644 --- a/src/module.c +++ b/src/module.c @@ -6526,24 +6526,32 @@ void RM_ScanCursorDestroy(RedisModuleScanCursor *cursor) { zfree(cursor); } -/* Scan api that allows a module to scan all the keys and value in the selected db. +/* Scan API that allows a module to scan all the keys and value in + * the selected db. * * Callback for scan implementation. - * void scan_callback(RedisModuleCtx *ctx, RedisModuleString *keyname, RedisModuleKey *key, void *privdata); - * - ctx - the redis module context provided to for the scan. - * - keyname - owned by the caller and need to be retained if used after this function. - * - key - holds info on the key and value, it is provided as best effort, in some cases it might - * be NULL, in which case the user should (can) use RedisModule_OpenKey (and CloseKey too). - * when it is provided, it is owned by the caller and will be free when the callback returns. - * - privdata - the user data provided to RedisModule_Scan. + * void scan_callback(RedisModuleCtx *ctx, RedisModuleString *keyname, + * RedisModuleKey *key, void *privdata); + * ctx - the redis module context provided to for the scan. + * keyname - owned by the caller and need to be retained if used after this + * function. + * + * key - holds info on the key and value, it is provided as best effort, in + * some cases it might be NULL, in which case the user should (can) use + * RedisModule_OpenKey (and CloseKey too). + * when it is provided, it is owned by the caller and will be free when the + * callback returns. + * + * privdata - the user data provided to RedisModule_Scan. * * The way it should be used: * RedisModuleCursor *c = RedisModule_ScanCursorCreate(); * while(RedisModule_Scan(ctx, c, callback, privateData)); * RedisModule_ScanCursorDestroy(c); * - * It is also possible to use this API from another thread while the lock is acquired durring - * the actuall call to RM_Scan: + * It is also possible to use this API from another thread while the lock + * is acquired durring the actuall call to RM_Scan: + * * RedisModuleCursor *c = RedisModule_ScanCursorCreate(); * RedisModule_ThreadSafeContextLock(ctx); * while(RedisModule_Scan(ctx, c, callback, privateData)){ @@ -6553,8 +6561,9 @@ void RM_ScanCursorDestroy(RedisModuleScanCursor *cursor) { * } * RedisModule_ScanCursorDestroy(c); * - * The function will return 1 if there are more elements to scan and 0 otherwise, - * possibly setting errno if the call failed. + * The function will return 1 if there are more elements to scan and + * 0 otherwise, possibly setting errno if the call failed. + * * It is also possible to restart and existing cursor using RM_CursorRestart. * * IMPORTANT: This API is very similar to the Redis SCAN command from the From 7d85577ce1339319fcd2e59de5b43b399ce6f443 Mon Sep 17 00:00:00 2001 From: ShooterIT Date: Sat, 29 Feb 2020 18:28:41 +0800 Subject: [PATCH 0147/1280] Avoid compiler warnings --- src/acl.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/acl.c b/src/acl.c index b046785ff..efe6b96ad 100644 --- a/src/acl.c +++ b/src/acl.c @@ -1830,6 +1830,7 @@ void aclCommand(client *c) { case ACL_DENIED_CMD: reasonstr="command"; break; case ACL_DENIED_KEY: reasonstr="key"; break; case ACL_DENIED_AUTH: reasonstr="auth"; break; + default: reasonstr="unknown"; } addReplyBulkCString(c,reasonstr); From f09711fdb0044b28c8d16e7a76a5db485580c73e Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 2 Mar 2020 16:49:11 +0100 Subject: [PATCH 0148/1280] Use a smaller getkeys global buffer. The idea is that very few commands have a lot of keys, and when this happens the allocation time becomes neglegible. --- src/db.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/db.c b/src/db.c index 6c5d4b4d4..04e26c33b 100644 --- a/src/db.c +++ b/src/db.c @@ -1305,7 +1305,7 @@ int expireIfNeeded(redisDb *db, robj *key) { /* ----------------------------------------------------------------------------- * API to get key arguments from commands * ---------------------------------------------------------------------------*/ -#define MAX_KEYS_BUFFER 65536 +#define MAX_KEYS_BUFFER 256 static int getKeysTempBuffer[MAX_KEYS_BUFFER]; /* The base case is to use the keys position as given in the command table From 1d048a3b468518552e0d9fce068b57ffea783357 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 3 Mar 2020 14:58:11 +0100 Subject: [PATCH 0149/1280] Remove RDB files used for replication in persistence-less instances. --- src/replication.c | 48 ++++++++++++++++++++++++++++++++++++++++++++++- src/server.c | 8 ++++++++ src/server.h | 1 + 3 files changed, 56 insertions(+), 1 deletion(-) diff --git a/src/replication.c b/src/replication.c index c497051c8..e71b269d8 100644 --- a/src/replication.c +++ b/src/replication.c @@ -45,6 +45,11 @@ void replicationSendAck(void); void putSlaveOnline(client *slave); int cancelReplicationHandshake(void); +/* We take a global flag to remember if this instance generated an RDB + * because of replication, so that we can remove the RDB file in case + * the instance is configured to have no persistence. */ +int RDBGeneratedByReplication = 0; + /* --------------------------- Utility functions ---------------------------- */ /* Return the pointer to a string representing the slave ip:listening_port @@ -591,6 +596,10 @@ int startBgsaveForReplication(int mincapa) { retval = C_ERR; } + /* If we succeeded to start a BGSAVE with disk target, let's remember + * this fact, so that we can later delete the file if needed. */ + if (retval == C_OK && !socket_target) RDBGeneratedByReplication = 1; + /* If we failed to BGSAVE, remove the slaves waiting for a full * resynchronization from the list of slaves, inform them with * an error about what happened, close the connection ASAP. */ @@ -883,6 +892,36 @@ void putSlaveOnline(client *slave) { replicationGetSlaveName(slave)); } +/* We call this function periodically to remove an RDB file that was + * generated because of replication, in an instance that is otherwise + * without any persistence. We don't want instances without persistence + * to take RDB files around, this violates certain policies in certain + * environments. */ +void removeRDBUsedToSyncReplicas(void) { + if (allPersistenceDisabled() && RDBGeneratedByReplication) { + client *slave; + listNode *ln; + listIter li; + + int delrdb = 1; + listRewind(server.slaves,&li); + while((ln = listNext(&li))) { + slave = ln->value; + if (slave->replstate == SLAVE_STATE_WAIT_BGSAVE_START || + slave->replstate == SLAVE_STATE_WAIT_BGSAVE_END || + slave->replstate == SLAVE_STATE_SEND_BULK) + { + delrdb = 0; + break; /* No need to check the other replicas. */ + } + } + if (delrdb) { + RDBGeneratedByReplication = 0; + unlink(server.rdb_filename); + } + } +} + void sendBulkToSlave(connection *conn) { client *slave = connGetPrivateData(conn); char buf[PROTO_IOBUF_LEN]; @@ -894,7 +933,8 @@ void sendBulkToSlave(connection *conn) { if (slave->replpreamble) { nwritten = connWrite(conn,slave->replpreamble,sdslen(slave->replpreamble)); if (nwritten == -1) { - serverLog(LL_VERBOSE,"Write error sending RDB preamble to replica: %s", + serverLog(LL_VERBOSE, + "Write error sending RDB preamble to replica: %s", connGetLastError(conn)); freeClient(slave); return; @@ -1639,12 +1679,14 @@ void readSyncBulkPayload(connection *conn) { "Failed trying to load the MASTER synchronization " "DB from disk"); cancelReplicationHandshake(); + if (allPersistenceDisabled()) unlink(server.rdb_filename); /* Note that there's no point in restarting the AOF on sync failure, it'll be restarted when sync succeeds or replica promoted. */ return; } /* Cleanup. */ + if (allPersistenceDisabled()) unlink(server.rdb_filename); zfree(server.repl_transfer_tmpfile); close(server.repl_transfer_fd); server.repl_transfer_fd = -1; @@ -3149,6 +3191,10 @@ void replicationCron(void) { } } + /* Remove the RDB file used for replication if Redis is not running + * with any persistence. */ + removeRDBUsedToSyncReplicas(); + /* Refresh the number of slaves with lag <= min-slaves-max-lag. */ refreshGoodSlavesCount(); replication_cron_loops++; /* Incremented with frequency 1 HZ. */ diff --git a/src/server.c b/src/server.c index bb8b3b103..a6d4b357e 100644 --- a/src/server.c +++ b/src/server.c @@ -1455,12 +1455,20 @@ void updateDictResizePolicy(void) { dictDisableResize(); } +/* Return true if there are no active children processes doing RDB saving, + * AOF rewriting, or some side process spawned by a loaded module. */ int hasActiveChildProcess() { return server.rdb_child_pid != -1 || server.aof_child_pid != -1 || server.module_child_pid != -1; } +/* Return true if this instance has persistence completely turned off: + * both RDB and AOF are disabled. */ +int allPersistenceDisabled(void) { + return server.saveparamslen == 0 && server.aof_state == AOF_OFF; +} + /* ======================= Cron: called every 100 ms ======================== */ /* Add a sample to the operations per second array of samples. */ diff --git a/src/server.h b/src/server.h index 87c293c26..5763671e4 100644 --- a/src/server.h +++ b/src/server.h @@ -1786,6 +1786,7 @@ void loadingProgress(off_t pos); void stopLoading(int success); void startSaving(int rdbflags); void stopSaving(int success); +int allPersistenceDisabled(void); #define DISK_ERROR_TYPE_AOF 1 /* Don't accept writes: AOF errors. */ #define DISK_ERROR_TYPE_RDB 2 /* Don't accept writes: RDB errors. */ From 31fe892339095051a828a030549d98b0394f744a Mon Sep 17 00:00:00 2001 From: Jamie Scott Date: Tue, 3 Mar 2020 18:03:16 -0800 Subject: [PATCH 0150/1280] Update Redis.conf to improve TLS usability When using TLS with a Redis.conf file the line for TLS reading tls-cert-file redis.crt tls-key-file redis.key is interpreted as one complete directive. I am separating this on two separate lines to improve usability so users do not get the below error. ubuntu@ip-172-31-29-250:~/redis-6.0-rc1$ ./src/redis-server redis.conf *** FATAL CONFIG FILE ERROR *** Reading the configuration file, at line 145 >>> 'tls-cert-file redis.crt tls-key-file redis.key' wrong number of arguments ubuntu@ip-172-31-29-250:~/redis-6.0-rc1$ vi redis.conf ubuntu@ip-172-31-29-250:~/redis-6.0-rc1$ ./src/redis-server redis.conf 23085:C 04 Mar 2020 01:58:12.631 # oO0OoO0OoO0Oo Redis is starting oO0OoO0OoO0Oo 23085:C 04 Mar 2020 01:58:12.631 # Redis version=5.9.101, bits=64, commit=00000000, modified=0, pid=23085, just started 23085:C 04 Mar 2020 01:58:12.631 # Configuration loaded 23085:M 04 Mar 2020 01:58:12.632 * Increased maximum number of open files to 10032 (it was originally set to 1024). --- redis.conf | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/redis.conf b/redis.conf index c04880f32..048b16302 100644 --- a/redis.conf +++ b/redis.conf @@ -142,7 +142,8 @@ tcp-keepalive 300 # server to connected clients, masters or cluster peers. These files should be # PEM formatted. # -# tls-cert-file redis.crt tls-key-file redis.key +# tls-cert-file redis.crt +# tls-key-file redis.key # Configure a DH parameters file to enable Diffie-Hellman (DH) key exchange: # From b661704fb51a43d09a6103d7cd5ad5d4f050e564 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 4 Mar 2020 11:10:54 +0100 Subject: [PATCH 0151/1280] Introduce bg_unlink(). --- src/replication.c | 34 +++++++++++++++++++++++++++++++--- 1 file changed, 31 insertions(+), 3 deletions(-) diff --git a/src/replication.c b/src/replication.c index e71b269d8..8a05b9696 100644 --- a/src/replication.c +++ b/src/replication.c @@ -79,6 +79,34 @@ char *replicationGetSlaveName(client *c) { return buf; } +/* Plain unlink() can block for quite some time in order to actually apply + * the file deletion to the filesystem. This call removes the file in a + * background thread instead. We actually just do close() in the thread, + * by using the fact that if there is another instance of the same file open, + * the foreground unlink() will not really do anything, and deleting the + * file will only happen once the last reference is lost. */ +int bg_unlink(const char *filename) { + int fd = open(filename,O_RDONLY|O_NONBLOCK); + if (fd == -1) { + /* Can't open the file? Fall back to unlinking in the main thread. */ + return unlink(filename); + } else { + /* The following unlink() will not do anything since file + * is still open. */ + int retval = unlink(filename); + if (retval == -1) { + /* If we got an unlink error, we just return it, closing the + * new reference we have to the file. */ + int old_errno = errno; + close(fd); /* This would overwrite our errno. So we saved it. */ + errno = old_errno; + return -1; + } + bioCreateBackgroundJob(BIO_CLOSE_FILE,(void*)(long)fd,NULL,NULL); + return 0; /* Success. */ + } +} + /* ---------------------------------- MASTER -------------------------------- */ void createReplicationBacklog(void) { @@ -917,7 +945,7 @@ void removeRDBUsedToSyncReplicas(void) { } if (delrdb) { RDBGeneratedByReplication = 0; - unlink(server.rdb_filename); + bg_unlink(server.rdb_filename); } } } @@ -1679,14 +1707,14 @@ void readSyncBulkPayload(connection *conn) { "Failed trying to load the MASTER synchronization " "DB from disk"); cancelReplicationHandshake(); - if (allPersistenceDisabled()) unlink(server.rdb_filename); + if (allPersistenceDisabled()) bg_unlink(server.rdb_filename); /* Note that there's no point in restarting the AOF on sync failure, it'll be restarted when sync succeeds or replica promoted. */ return; } /* Cleanup. */ - if (allPersistenceDisabled()) unlink(server.rdb_filename); + if (allPersistenceDisabled()) bg_unlink(server.rdb_filename); zfree(server.repl_transfer_tmpfile); close(server.repl_transfer_fd); server.repl_transfer_fd = -1; From 994b992782cb8b1bbe361ca83ebb5cf3509927a8 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 4 Mar 2020 11:19:55 +0100 Subject: [PATCH 0152/1280] Log RDB deletion in persistence-less instances. --- src/replication.c | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/src/replication.c b/src/replication.c index 8a05b9696..acecdd098 100644 --- a/src/replication.c +++ b/src/replication.c @@ -944,6 +944,8 @@ void removeRDBUsedToSyncReplicas(void) { } } if (delrdb) { + serverLog(LL_NOTICE,"Removing the RDB file used to feed replicas " + "in a persistence-less instance"); RDBGeneratedByReplication = 0; bg_unlink(server.rdb_filename); } @@ -1707,14 +1709,25 @@ void readSyncBulkPayload(connection *conn) { "Failed trying to load the MASTER synchronization " "DB from disk"); cancelReplicationHandshake(); - if (allPersistenceDisabled()) bg_unlink(server.rdb_filename); + if (allPersistenceDisabled()) { + serverLog(LL_NOTICE,"Removing the RDB file obtained from " + "the master. This replica has persistence " + "disabled"); + bg_unlink(server.rdb_filename); + } /* Note that there's no point in restarting the AOF on sync failure, it'll be restarted when sync succeeds or replica promoted. */ return; } /* Cleanup. */ - if (allPersistenceDisabled()) bg_unlink(server.rdb_filename); + if (allPersistenceDisabled()) { + serverLog(LL_NOTICE,"Removing the RDB file obtained from " + "the master. This replica has persistence " + "disabled"); + bg_unlink(server.rdb_filename); + } + zfree(server.repl_transfer_tmpfile); close(server.repl_transfer_fd); server.repl_transfer_fd = -1; From dff91688a6333611f10120e3c22b0a190b948489 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 4 Mar 2020 12:55:49 +0100 Subject: [PATCH 0153/1280] Check that the file exists in removeRDBUsedToSyncReplicas(). --- src/replication.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/replication.c b/src/replication.c index acecdd098..20666bd20 100644 --- a/src/replication.c +++ b/src/replication.c @@ -944,10 +944,14 @@ void removeRDBUsedToSyncReplicas(void) { } } if (delrdb) { - serverLog(LL_NOTICE,"Removing the RDB file used to feed replicas " - "in a persistence-less instance"); - RDBGeneratedByReplication = 0; - bg_unlink(server.rdb_filename); + struct stat sb; + if (lstat(server.rdb_filename,&sb) != -1) { + RDBGeneratedByReplication = 0; + serverLog(LL_NOTICE, + "Removing the RDB file used to feed replicas " + "in a persistence-less instance"); + bg_unlink(server.rdb_filename); + } } } } From e0cf09cacef512a83048eef4ef52261e68cd326c Mon Sep 17 00:00:00 2001 From: "bodong.ybd" Date: Wed, 4 Mar 2020 20:51:45 +0800 Subject: [PATCH 0154/1280] Added BITFIELD_RO variants for read-only operations. --- src/bitops.c | 19 ++++++++++++++++++- src/server.c | 4 ++++ src/server.h | 1 + tests/unit/bitfield.tcl | 31 +++++++++++++++++++++++++++++++ 4 files changed, 54 insertions(+), 1 deletion(-) diff --git a/src/bitops.c b/src/bitops.c index ee1ce0460..ffb330013 100644 --- a/src/bitops.c +++ b/src/bitops.c @@ -902,6 +902,9 @@ void bitposCommand(client *c) { * OVERFLOW [WRAP|SAT|FAIL] */ +#define BITFIELD_COMMON (1<<0) +#define BITFIELD_READONLY (1<<1) + struct bitfieldOp { uint64_t offset; /* Bitfield offset. */ int64_t i64; /* Increment amount (INCRBY) or SET value */ @@ -911,7 +914,7 @@ struct bitfieldOp { int sign; /* True if signed, otherwise unsigned op. */ }; -void bitfieldCommand(client *c) { +void bitfieldGeneric(client *c, int flags) { robj *o; size_t bitoffset; int j, numops = 0, changes = 0; @@ -999,6 +1002,12 @@ void bitfieldCommand(client *c) { return; } } else { + if (flags & BITFIELD_READONLY) { + zfree(ops); + addReplyError(c, "bitfield_ro only support get subcommand"); + return; + } + /* Lookup by making room up to the farest bit reached by * this operation. */ if ((o = lookupStringForBitCommand(c, @@ -1129,3 +1138,11 @@ void bitfieldCommand(client *c) { } zfree(ops); } + +void bitfieldCommand(client *c) { + bitfieldGeneric(c, BITFIELD_COMMON); +} + +void bitfieldroCommand(client *c) { + bitfieldGeneric(c, BITFIELD_READONLY); +} diff --git a/src/server.c b/src/server.c index bb8b3b103..ab2afd47f 100644 --- a/src/server.c +++ b/src/server.c @@ -238,6 +238,10 @@ struct redisCommand redisCommandTable[] = { "write use-memory @bitmap", 0,NULL,1,1,1,0,0,0}, + {"bitfield_ro",bitfieldroCommand,-2, + "read-only fast @bitmap", + 0,NULL,1,1,1,0,0,0}, + {"setrange",setrangeCommand,4, "write use-memory @string", 0,NULL,1,1,1,0,0,0}, diff --git a/src/server.h b/src/server.h index 87c293c26..e5043855c 100644 --- a/src/server.h +++ b/src/server.h @@ -2171,6 +2171,7 @@ void existsCommand(client *c); void setbitCommand(client *c); void getbitCommand(client *c); void bitfieldCommand(client *c); +void bitfieldroCommand(client *c); void setrangeCommand(client *c); void getrangeCommand(client *c); void incrCommand(client *c); diff --git a/tests/unit/bitfield.tcl b/tests/unit/bitfield.tcl index d76452b1b..819d8f36d 100644 --- a/tests/unit/bitfield.tcl +++ b/tests/unit/bitfield.tcl @@ -199,3 +199,34 @@ start_server {tags {"bitops"}} { r del mystring } } + +start_server {tags {"repl"}} { + start_server {} { + set master [srv -1 client] + set master_host [srv -1 host] + set master_port [srv -1 port] + set slave [srv 0 client] + + test {setup slave} { + $slave slaveof $master_host $master_port + wait_for_condition 50 100 { + [s 0 master_link_status] eq {up} + } else { + fail "Replication not started." + } + } + + test {write on master, read on slave} { + $master del bits + assert_equal 0 [$master bitfield bits set u8 0 255] + assert_equal 255 [$master bitfield bits set u8 0 100] + wait_for_ofs_sync $master $slave + assert_equal 100 [$slave bitfield_ro bits get u8 0] + } + + test {bitfield_ro with write option} { + catch {$slave bitfield_ro bits set u8 0 100 get u8 0} err + assert_match {*ERR bitfield_ro only support get subcommand*} $err + } + } +} From b3134c8dd3be0ac1d617d15271b5276ee91a144e Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 4 Mar 2020 17:44:21 +0100 Subject: [PATCH 0155/1280] Make sync RDB deletion configurable. Default to no. --- src/config.c | 1 + src/replication.c | 23 +++++++++++++++++++---- src/server.h | 2 ++ 3 files changed, 22 insertions(+), 4 deletions(-) diff --git a/src/config.c b/src/config.c index fd04b7c87..211b6d003 100644 --- a/src/config.c +++ b/src/config.c @@ -2084,6 +2084,7 @@ standardConfig configs[] = { createBoolConfig("always-show-logo", NULL, IMMUTABLE_CONFIG, server.always_show_logo, 0, NULL, NULL), createBoolConfig("protected-mode", NULL, MODIFIABLE_CONFIG, server.protected_mode, 1, NULL, NULL), createBoolConfig("rdbcompression", NULL, MODIFIABLE_CONFIG, server.rdb_compression, 1, NULL, NULL), + createBoolConfig("rdb-del-sync-files", NULL, MODIFIABLE_CONFIG, server.rdb_del_sync_files, 0, NULL, NULL), createBoolConfig("activerehashing", NULL, MODIFIABLE_CONFIG, server.activerehashing, 1, NULL, NULL), createBoolConfig("stop-writes-on-bgsave-error", NULL, MODIFIABLE_CONFIG, server.stop_writes_on_bgsave_err, 1, NULL, NULL), createBoolConfig("dynamic-hz", NULL, MODIFIABLE_CONFIG, server.dynamic_hz, 1, NULL, NULL), /* Adapt hz to # of clients.*/ diff --git a/src/replication.c b/src/replication.c index 20666bd20..429e1d8a8 100644 --- a/src/replication.c +++ b/src/replication.c @@ -625,8 +625,12 @@ int startBgsaveForReplication(int mincapa) { } /* If we succeeded to start a BGSAVE with disk target, let's remember - * this fact, so that we can later delete the file if needed. */ - if (retval == C_OK && !socket_target) RDBGeneratedByReplication = 1; + * this fact, so that we can later delete the file if needed. Note + * that we don't set the flag to 1 if the feature is disabled, otherwise + * it would never be cleared: the file is not deleted. This way if + * the user enables it later with CONFIG SET, we are fine. */ + if (retval == C_OK && !socket_target && server.rdb_del_sync_files) + RDBGeneratedByReplication = 1; /* If we failed to BGSAVE, remove the slaves waiting for a full * resynchronization from the list of slaves, inform them with @@ -926,6 +930,17 @@ void putSlaveOnline(client *slave) { * to take RDB files around, this violates certain policies in certain * environments. */ void removeRDBUsedToSyncReplicas(void) { + /* If the feature is disabled, return ASAP but also clear the + * RDBGeneratedByReplication flag in case it was set. Otherwise if the + * feature was enabled, but gets disabled later with CONFIG SET, the + * flag may remain set to one: then next time the feature is re-enabled + * via CONFIG SET we have have it set even if no RDB was generated + * because of replication recently. */ + if (!server.rdb_del_sync_files) { + RDBGeneratedByReplication = 0; + return; + } + if (allPersistenceDisabled() && RDBGeneratedByReplication) { client *slave; listNode *ln; @@ -1713,7 +1728,7 @@ void readSyncBulkPayload(connection *conn) { "Failed trying to load the MASTER synchronization " "DB from disk"); cancelReplicationHandshake(); - if (allPersistenceDisabled()) { + if (server.rdb_del_sync_files && allPersistenceDisabled()) { serverLog(LL_NOTICE,"Removing the RDB file obtained from " "the master. This replica has persistence " "disabled"); @@ -1725,7 +1740,7 @@ void readSyncBulkPayload(connection *conn) { } /* Cleanup. */ - if (allPersistenceDisabled()) { + if (server.rdb_del_sync_files && allPersistenceDisabled()) { serverLog(LL_NOTICE,"Removing the RDB file obtained from " "the master. This replica has persistence " "disabled"); diff --git a/src/server.h b/src/server.h index 5763671e4..3c19a17ea 100644 --- a/src/server.h +++ b/src/server.h @@ -1202,6 +1202,8 @@ struct redisServer { char *rdb_filename; /* Name of RDB file */ int rdb_compression; /* Use compression in RDB? */ int rdb_checksum; /* Use RDB checksum? */ + int rdb_del_sync_files; /* Remove RDB files used only for SYNC if + the instance does not use persistence. */ time_t lastsave; /* Unix time of last successful save */ time_t lastbgsave_try; /* Unix time of last attempted bgsave */ time_t rdb_save_time_last; /* Time used by last RDB save run. */ From 40b1414e6fb5ee53b5a5a5ac39ccf96cbcbaee6c Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 4 Mar 2020 17:58:05 +0100 Subject: [PATCH 0156/1280] RDB deletion: document it in example redis.conf. --- redis.conf | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/redis.conf b/redis.conf index c04880f32..8609a9f57 100644 --- a/redis.conf +++ b/redis.conf @@ -321,6 +321,19 @@ rdbchecksum yes # The filename where to dump the DB dbfilename dump.rdb +# Remove RDB files used by replication in instances without persistence +# enabled. By default this option is disabled, however there are environments +# where for regulations or other security concerns, RDB files persisted on +# disk by masters in order to feed replicas, or stored on disk by replicas +# in order to load them for the initial synchronization, should be deleted +# ASAP. Note that this option ONLY WORKS in instances that have both AOF +# and RDB persistence disabled, otherwise is completely ignored. +# +# An alternative (and sometimes better) way to obtain the same effect is +# to use diskless replication on both master and replicas instances. However +# in the case of replicas, diskless is not always an option. +rdb-del-sync-files no + # The working directory. # # The DB will be written inside this directory, with the filename specified From a97330a70e1677eddbdd6ded2dc0a85d509abfe1 Mon Sep 17 00:00:00 2001 From: lifubang Date: Thu, 5 Mar 2020 18:13:43 +0800 Subject: [PATCH 0157/1280] update linenoise to https://github.com/antirez/linenoise/tree/fc9667a81d43911a6690fb1e68c16e6e3bb8df05 Signed-off-by: lifubang --- deps/linenoise/README.markdown | 25 ++++++++++++++++++++++++- deps/linenoise/example.c | 5 +++++ deps/linenoise/linenoise.c | 31 ++++++++++++++++++++++++++++--- deps/linenoise/linenoise.h | 2 ++ 4 files changed, 59 insertions(+), 4 deletions(-) diff --git a/deps/linenoise/README.markdown b/deps/linenoise/README.markdown index e01642cf8..1afea2ae6 100644 --- a/deps/linenoise/README.markdown +++ b/deps/linenoise/README.markdown @@ -21,7 +21,7 @@ So what usually happens is either: The result is a pollution of binaries without line editing support. -So I spent more or less two hours doing a reality check resulting in this little library: is it *really* needed for a line editing library to be 20k lines of code? Apparently not, it is possibe to get a very small, zero configuration, trivial to embed library, that solves the problem. Smaller programs will just include this, supporing line editing out of the box. Larger programs may use this little library or just checking with configure if readline/libedit is available and resorting to Linenoise if not. +So I spent more or less two hours doing a reality check resulting in this little library: is it *really* needed for a line editing library to be 20k lines of code? Apparently not, it is possibe to get a very small, zero configuration, trivial to embed library, that solves the problem. Smaller programs will just include this, supporting line editing out of the box. Larger programs may use this little library or just checking with configure if readline/libedit is available and resorting to Linenoise if not. ## Terminals, in 2010. @@ -126,6 +126,24 @@ Linenoise has direct support for persisting the history into an history file. The functions `linenoiseHistorySave` and `linenoiseHistoryLoad` do just that. Both functions return -1 on error and 0 on success. +## Mask mode + +Sometimes it is useful to allow the user to type passwords or other +secrets that should not be displayed. For such situations linenoise supports +a "mask mode" that will just replace the characters the user is typing +with `*` characters, like in the following example: + + $ ./linenoise_example + hello> get mykey + echo: 'get mykey' + hello> /mask + hello> ********* + +You can enable and disable mask mode using the following two functions: + + void linenoiseMaskModeEnable(void); + void linenoiseMaskModeDisable(void); + ## Completion Linenoise supports completion, which is the ability to complete the user @@ -222,3 +240,8 @@ Sometimes you may want to clear the screen as a result of something the user typed. You can do this by calling the following function: void linenoiseClearScreen(void); + +## Related projects + +* [Linenoise NG](https://github.com/arangodb/linenoise-ng) is a fork of Linenoise that aims to add more advanced features like UTF-8 support, Windows support and other features. Uses C++ instead of C as development language. +* [Linenoise-swift](https://github.com/andybest/linenoise-swift) is a reimplementation of Linenoise written in Swift. diff --git a/deps/linenoise/example.c b/deps/linenoise/example.c index 3a544d3c6..74358c323 100644 --- a/deps/linenoise/example.c +++ b/deps/linenoise/example.c @@ -55,6 +55,7 @@ int main(int argc, char **argv) { * * The typed string is returned as a malloc() allocated string by * linenoise, so the user needs to free() it. */ + while((line = linenoise("hello> ")) != NULL) { /* Do something with the string. */ if (line[0] != '\0' && line[0] != '/') { @@ -65,6 +66,10 @@ int main(int argc, char **argv) { /* The "/historylen" command will change the history len. */ int len = atoi(line+11); linenoiseHistorySetMaxLen(len); + } else if (!strncmp(line, "/mask", 5)) { + linenoiseMaskModeEnable(); + } else if (!strncmp(line, "/unmask", 7)) { + linenoiseMaskModeDisable(); } else if (line[0] == '/') { printf("Unreconized command: %s\n", line); } diff --git a/deps/linenoise/linenoise.c b/deps/linenoise/linenoise.c index fce14a7c5..01c3b9350 100644 --- a/deps/linenoise/linenoise.c +++ b/deps/linenoise/linenoise.c @@ -125,6 +125,7 @@ static linenoiseHintsCallback *hintsCallback = NULL; static linenoiseFreeHintsCallback *freeHintsCallback = NULL; static struct termios orig_termios; /* In order to restore at exit.*/ +static int maskmode = 0; /* Show "***" instead of input. For passwords. */ static int rawmode = 0; /* For atexit() function to check if restore is needed*/ static int mlmode = 0; /* Multi line mode. Default is single line. */ static int atexit_registered = 0; /* Register atexit just 1 time. */ @@ -197,6 +198,19 @@ FILE *lndebug_fp = NULL; /* ======================= Low level terminal handling ====================== */ +/* Enable "mask mode". When it is enabled, instead of the input that + * the user is typing, the terminal will just display a corresponding + * number of asterisks, like "****". This is useful for passwords and other + * secrets that should not be displayed. */ +void linenoiseMaskModeEnable(void) { + maskmode = 1; +} + +/* Disable mask mode. */ +void linenoiseMaskModeDisable(void) { + maskmode = 0; +} + /* Set if to use or not the multi line mode. */ void linenoiseSetMultiLine(int ml) { mlmode = ml; @@ -485,6 +499,8 @@ void refreshShowHints(struct abuf *ab, struct linenoiseState *l, int plen) { if (bold == 1 && color == -1) color = 37; if (color != -1 || bold != 0) snprintf(seq,64,"\033[%d;%d;49m",bold,color); + else + seq[0] = '\0'; abAppend(ab,seq,strlen(seq)); abAppend(ab,hint,hintlen); if (color != -1 || bold != 0) @@ -523,7 +539,11 @@ static void refreshSingleLine(struct linenoiseState *l) { abAppend(&ab,seq,strlen(seq)); /* Write the prompt and the current buffer content */ abAppend(&ab,l->prompt,strlen(l->prompt)); - abAppend(&ab,buf,len); + if (maskmode == 1) { + while (len--) abAppend(&ab,"*",1); + } else { + abAppend(&ab,buf,len); + } /* Show hits if any. */ refreshShowHints(&ab,l,plen); /* Erase to right */ @@ -577,7 +597,11 @@ static void refreshMultiLine(struct linenoiseState *l) { /* Write the prompt and the current buffer content */ abAppend(&ab,l->prompt,strlen(l->prompt)); - abAppend(&ab,l->buf,l->len); + if (maskmode == 1) { + for (uint i = 0; i < l->len; i++) abAppend(&ab,"*",1); + } else { + abAppend(&ab,l->buf,l->len); + } /* Show hits if any. */ refreshShowHints(&ab,l,plen); @@ -645,7 +669,8 @@ int linenoiseEditInsert(struct linenoiseState *l, char c) { if ((!mlmode && l->plen+l->len < l->cols && !hintsCallback)) { /* Avoid a full update of the line in the * trivial case. */ - if (write(l->ofd,&c,1) == -1) return -1; + char d = (maskmode==1) ? '*' : c; + if (write(l->ofd,&d,1) == -1) return -1; } else { refreshLine(l); } diff --git a/deps/linenoise/linenoise.h b/deps/linenoise/linenoise.h index ed20232c5..6dfee73bc 100644 --- a/deps/linenoise/linenoise.h +++ b/deps/linenoise/linenoise.h @@ -65,6 +65,8 @@ int linenoiseHistoryLoad(const char *filename); void linenoiseClearScreen(void); void linenoiseSetMultiLine(int ml); void linenoisePrintKeyCodes(void); +void linenoiseMaskModeEnable(void); +void linenoiseMaskModeDisable(void); #ifdef __cplusplus } From 45570731b931f4354657f602e1d5a75053e8c1fe Mon Sep 17 00:00:00 2001 From: lifubang Date: Thu, 5 Mar 2020 18:17:32 +0800 Subject: [PATCH 0158/1280] add askpass mode Signed-off-by: lifubang --- src/redis-cli.c | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/src/redis-cli.c b/src/redis-cli.c index 54898f42e..b44db9a1e 100644 --- a/src/redis-cli.c +++ b/src/redis-cli.c @@ -229,6 +229,7 @@ static struct config { int hotkeys; int stdinarg; /* get last arg from stdin. (-x option) */ char *auth; + int askpass; char *user; int output; /* output mode, see OUTPUT_* defines */ sds mb_delim; @@ -1450,6 +1451,8 @@ static int parseOptions(int argc, char **argv) { config.dbnum = atoi(argv[++i]); } else if (!strcmp(argv[i], "--no-auth-warning")) { config.no_auth_warning = 1; + } else if (!strcmp(argv[i], "--askpass")) { + config.askpass = 1; } else if ((!strcmp(argv[i],"-a") || !strcmp(argv[i],"--pass")) && !lastarg) { @@ -1690,6 +1693,9 @@ static void usage(void) { " (if both are used, this argument takes predecence).\n" " --user Used to send ACL style 'AUTH username pass'. Needs -a.\n" " --pass Alias of -a for consistency with the new --user option.\n" +" --askpass Force user to input password with mask from STDIN.\n" +" If this argument is used, '-a' and " REDIS_CLI_AUTH_ENV "\n" +" environment variable will be ignored.\n" " -u Server URI.\n" " -r Execute specified command N times.\n" " -i When -r is used, waits seconds per command.\n" @@ -7858,6 +7864,13 @@ static void intrinsicLatencyMode(void) { } } +static sds askPassword() { + linenoiseMaskModeEnable(); + sds auth = linenoise("Please input password: "); + linenoiseMaskModeDisable(); + return auth; +} + /*------------------------------------------------------------------------------ * Program main() *--------------------------------------------------------------------------- */ @@ -7894,6 +7907,7 @@ int main(int argc, char **argv) { config.hotkeys = 0; config.stdinarg = 0; config.auth = NULL; + config.askpass = 0; config.user = NULL; config.eval = NULL; config.eval_ldb = 0; @@ -7935,6 +7949,10 @@ int main(int argc, char **argv) { parseEnv(); + if (config.askpass) { + config.auth = askPassword(); + } + #ifdef USE_OPENSSL if (config.tls) { ERR_load_crypto_strings(); @@ -8044,4 +8062,4 @@ int main(int argc, char **argv) { } else { return noninteractive(argc,convertToSds(argc,argv)); } -} +} \ No newline at end of file From d18e79f91dde494e7f1402f42bfa1bff5b3490df Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Thu, 5 Mar 2020 16:55:14 +0200 Subject: [PATCH 0159/1280] fix for flaky psync2 test *** [err]: PSYNC2: total sum of full synchronizations is exactly 4 in tests/integration/psync2.tcl Expected 5 == 4 (context: type eval line 6 cmd {assert {$sum == 4}} proc ::test) issue was that sometime the test got an unexpected full sync since it tried to switch to the replica before it was in sync with it's master. --- tests/integration/psync2.tcl | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tests/integration/psync2.tcl b/tests/integration/psync2.tcl index d1212b640..333736ffa 100644 --- a/tests/integration/psync2.tcl +++ b/tests/integration/psync2.tcl @@ -114,6 +114,27 @@ start_server {} { } } + # wait for all the slaves to be in sync with the master + set master_ofs [status $R($master_id) master_repl_offset] + wait_for_condition 500 100 { + $master_ofs == [status $R(0) master_repl_offset] && + $master_ofs == [status $R(1) master_repl_offset] && + $master_ofs == [status $R(2) master_repl_offset] && + $master_ofs == [status $R(3) master_repl_offset] && + $master_ofs == [status $R(4) master_repl_offset] + } else { + if {$debug_msg} { + for {set j 0} {$j < 5} {incr j} { + puts "$j: sync_full: [status $R($j) sync_full]" + puts "$j: id1 : [status $R($j) master_replid]:[status $R($j) master_repl_offset]" + puts "$j: id2 : [status $R($j) master_replid2]:[status $R($j) second_repl_offset]" + puts "$j: backlog : firstbyte=[status $R($j) repl_backlog_first_byte_offset] len=[status $R($j) repl_backlog_histlen]" + puts "---" + } + } + fail "Slaves are not in sync with the master after too long time." + } + # Put down the old master so that it cannot generate more # replication stream, this way in the next master switch, the time at # which we move slaves away is not important, each will have full From b349d319d693299a5894188dd07fa8b894207976 Mon Sep 17 00:00:00 2001 From: David Carlier Date: Sat, 7 Mar 2020 10:43:41 +0000 Subject: [PATCH 0160/1280] debug, dump registers on arm too. --- src/debug.c | 82 +++++++++++++++++++++++++++++++++++------------------ 1 file changed, 55 insertions(+), 27 deletions(-) diff --git a/src/debug.c b/src/debug.c index 36af35aec..a3e434d21 100644 --- a/src/debug.c +++ b/src/debug.c @@ -1045,6 +1045,61 @@ void logRegisters(ucontext_t *uc) { (unsigned long) uc->uc_mcontext.gregs[18] ); logStackContent((void**)uc->uc_mcontext.gregs[15]); + #elif defined(__aarch64__) /* Linux AArch64 */ + serverLog(LL_WARNING, + "\n" + "X18:%016lx X19:%016lx\nX20:%016lx X21:%016lx\n" + "X22:%016lx X23:%016lx\nX24:%016lx X25:%016lx\n" + "X26:%016lx X27:%016lx\nX28:%016lx X29:%016lx\n" + "X30:%016lx\n" + "pc:%016lx sp:%016lx\npstate:%016lx fault_address:%016lx\n", + (unsigned long) uc->uc_mcontext.regs[18], + (unsigned long) uc->uc_mcontext.regs[19], + (unsigned long) uc->uc_mcontext.regs[20], + (unsigned long) uc->uc_mcontext.regs[21], + (unsigned long) uc->uc_mcontext.regs[22], + (unsigned long) uc->uc_mcontext.regs[23], + (unsigned long) uc->uc_mcontext.regs[24], + (unsigned long) uc->uc_mcontext.regs[25], + (unsigned long) uc->uc_mcontext.regs[26], + (unsigned long) uc->uc_mcontext.regs[27], + (unsigned long) uc->uc_mcontext.regs[28], + (unsigned long) uc->uc_mcontext.regs[29], + (unsigned long) uc->uc_mcontext.regs[30], + (unsigned long) uc->uc_mcontext.pc, + (unsigned long) uc->uc_mcontext.sp, + (unsigned long) uc->uc_mcontext.pstate, + (unsigned long) uc->uc_mcontext.fault_address + ); + logStackContent((void**)uc->uc_mcontext.sp); + #elif defined(__arm__) /* Linux ARM */ + serverLog(LL_WARNING, + "\n" + "R10:%016lx R9 :%016lx\nR8 :%016lx R7 :%016lx\n" + "R6 :%016lx R5 :%016lx\nR4 :%016lx R3 :%016lx\n" + "R2 :%016lx R1 :%016lx\nR0 :%016lx EC :%016lx\n" + "fp: %016lx ip:%016lx\n", + "pc:%016lx sp:%016lx\ncpsr:%016lx fault_address:%016lx\n", + (unsigned long) uc->uc_mcontext.arm_r10, + (unsigned long) uc->uc_mcontext.arm_r9, + (unsigned long) uc->uc_mcontext.arm_r8, + (unsigned long) uc->uc_mcontext.arm_r7, + (unsigned long) uc->uc_mcontext.arm_r6, + (unsigned long) uc->uc_mcontext.arm_r5, + (unsigned long) uc->uc_mcontext.arm_r4, + (unsigned long) uc->uc_mcontext.arm_r3, + (unsigned long) uc->uc_mcontext.arm_r2, + (unsigned long) uc->uc_mcontext.arm_r1, + (unsigned long) uc->uc_mcontext.arm_r0, + (unsigned long) uc->uc_mcontext.error_code, + (unsigned long) uc->uc_mcontext.arm_fp, + (unsigned long) uc->uc_mcontext.arm_ip, + (unsigned long) uc->uc_mcontext.arm_pc, + (unsigned long) uc->uc_mcontext.arm_sp, + (unsigned long) uc->uc_mcontext.arm_cpsr, + (unsigned long) uc->uc_mcontext.fault_address + ); + logStackContent((void**)uc->uc_mcontext.arm_sp); #endif #elif defined(__FreeBSD__) #if defined(__x86_64__) @@ -1185,33 +1240,6 @@ void logRegisters(ucontext_t *uc) { (unsigned long) uc->uc_mcontext.mc_cs ); logStackContent((void**)uc->uc_mcontext.mc_rsp); -#elif defined(__aarch64__) /* Linux AArch64 */ - serverLog(LL_WARNING, - "\n" - "X18:%016lx X19:%016lx\nX20:%016lx X21:%016lx\n" - "X22:%016lx X23:%016lx\nX24:%016lx X25:%016lx\n" - "X26:%016lx X27:%016lx\nX28:%016lx X29:%016lx\n" - "X30:%016lx\n" - "pc:%016lx sp:%016lx\npstate:%016lx fault_address:%016lx\n", - (unsigned long) uc->uc_mcontext.regs[18], - (unsigned long) uc->uc_mcontext.regs[19], - (unsigned long) uc->uc_mcontext.regs[20], - (unsigned long) uc->uc_mcontext.regs[21], - (unsigned long) uc->uc_mcontext.regs[22], - (unsigned long) uc->uc_mcontext.regs[23], - (unsigned long) uc->uc_mcontext.regs[24], - (unsigned long) uc->uc_mcontext.regs[25], - (unsigned long) uc->uc_mcontext.regs[26], - (unsigned long) uc->uc_mcontext.regs[27], - (unsigned long) uc->uc_mcontext.regs[28], - (unsigned long) uc->uc_mcontext.regs[29], - (unsigned long) uc->uc_mcontext.regs[30], - (unsigned long) uc->uc_mcontext.pc, - (unsigned long) uc->uc_mcontext.sp, - (unsigned long) uc->uc_mcontext.pstate, - (unsigned long) uc->uc_mcontext.fault_address - ); - logStackContent((void**)uc->uc_mcontext.sp); #else serverLog(LL_WARNING, " Dumping of registers not supported for this OS/arch"); From 512badad1b97291eeaac350789300f8a231b01c8 Mon Sep 17 00:00:00 2001 From: guodongxiaren <879231132@qq.com> Date: Sat, 7 Mar 2020 19:38:27 +0800 Subject: [PATCH 0161/1280] string literal should be const char* --- src/asciilogo.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/asciilogo.h b/src/asciilogo.h index 83c538b54..044ca0c55 100644 --- a/src/asciilogo.h +++ b/src/asciilogo.h @@ -27,7 +27,7 @@ * POSSIBILITY OF SUCH DAMAGE. */ -char *ascii_logo = +const char *ascii_logo = " _._ \n" " _.-``__ ''-._ \n" " _.-`` `. `_. ''-._ Redis %s (%s/%d) %s bit\n" From 1bf0b5b1473f15df2d637dabce12a422c85630f4 Mon Sep 17 00:00:00 2001 From: Jamie Scott Date: Mon, 9 Mar 2020 12:53:44 -0700 Subject: [PATCH 0162/1280] Remove default guidance in Redis.conf Removing the default guidance in Redis.conf since this is not an available value. --- redis.conf | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/redis.conf b/redis.conf index 048b16302..a8e2fd89d 100644 --- a/redis.conf +++ b/redis.conf @@ -176,8 +176,7 @@ tcp-keepalive 300 # tls-cluster yes # Explicitly specify TLS versions to support. Allowed values are case insensitive -# and include "TLSv1", "TLSv1.1", "TLSv1.2", "TLSv1.3" (OpenSSL >= 1.1.1) or -# "default" which is currently >= TLSv1.1. +# and include "TLSv1", "TLSv1.1", "TLSv1.2", "TLSv1.3" (OpenSSL >= 1.1.1) # # tls-protocols TLSv1.2 From 53fd8f4d0dc1b9a7311485cc05703dc1c070745e Mon Sep 17 00:00:00 2001 From: "bodong.ybd" Date: Wed, 11 Mar 2020 20:55:51 +0800 Subject: [PATCH 0163/1280] Fix bug of tcl test using external server --- tests/support/server.tcl | 7 +++++-- tests/test_helper.tcl | 3 +++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/support/server.tcl b/tests/support/server.tcl index d086366dc..400017c5f 100644 --- a/tests/support/server.tcl +++ b/tests/support/server.tcl @@ -159,9 +159,12 @@ proc start_server {options {code undefined}} { if {$::external} { if {[llength $::servers] == 0} { set srv {} + # In test_server_main(tests/test_helper.tcl:215~218), increase the value of start_port + # and assign it to ::port through the `--port` option, so we need to reduce it. + set baseport [expr {$::port-100}] dict set srv "host" $::host - dict set srv "port" $::port - set client [redis $::host $::port 0 $::tls] + dict set srv "port" $baseport + set client [redis $::host $baseport 0 $::tls] dict set srv "client" $client $client select 9 diff --git a/tests/test_helper.tcl b/tests/test_helper.tcl index fa5579669..4dbead193 100644 --- a/tests/test_helper.tcl +++ b/tests/test_helper.tcl @@ -505,6 +505,9 @@ for {set j 0} {$j < [llength $argv]} {incr j} { } elseif {$opt eq {--host}} { set ::external 1 set ::host $arg + # If we use an external server, we can only set numclients to 1, + # otherwise the port will be miscalculated. + set ::numclients 1 incr j } elseif {$opt eq {--port}} { set ::port $arg From f9ad2cfd7504883fd96d91b5f51ed342337439fa Mon Sep 17 00:00:00 2001 From: Itamar Haber Date: Wed, 11 Mar 2020 18:43:03 +0200 Subject: [PATCH 0164/1280] Adds keyspace notifications to migrate and restore --- src/cluster.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/cluster.c b/src/cluster.c index c05e46f76..5f63d2b8f 100644 --- a/src/cluster.c +++ b/src/cluster.c @@ -4981,6 +4981,7 @@ void restoreCommand(client *c) { } objectSetLRUOrLFU(obj,lfu_freq,lru_idle,lru_clock,1000); signalModifiedKey(c->db,c->argv[1]); + notifyKeyspaceEvent(NOTIFY_GENERIC,"restore",c->argv[1],c->db->id); addReply(c,shared.ok); server.dirty++; } @@ -5327,6 +5328,7 @@ try_again: /* No COPY option: remove the local key, signal the change. */ dbDelete(c->db,kv[j]); signalModifiedKey(c->db,kv[j]); + notifyKeyspaceEvent(NOTIFY_GENERIC,"del",kv[j],c->db->id); server.dirty++; /* Populate the argument vector to replace the old one. */ @@ -5489,7 +5491,7 @@ void readwriteCommand(client *c) { * already "down" but it is fragile to rely on the update of the global state, * so we also handle it here. * - * CLUSTER_REDIR_DOWN_STATE and CLUSTER_REDIR_DOWN_RO_STATE if the cluster is + * CLUSTER_REDIR_DOWN_STATE and CLUSTER_REDIR_DOWN_RO_STATE if the cluster is * down but the user attempts to execute a command that addresses one or more keys. */ clusterNode *getNodeByQuery(client *c, struct redisCommand *cmd, robj **argv, int argc, int *hashslot, int *error_code) { clusterNode *n = NULL; From e7765e025c5cdc9ba0bc47665052eea4f523448e Mon Sep 17 00:00:00 2001 From: "bodong.ybd" Date: Thu, 12 Mar 2020 11:12:37 +0800 Subject: [PATCH 0165/1280] Remove duplicate obj files in Makefile --- src/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Makefile b/src/Makefile index 00b623a4b..bbfb06440 100644 --- a/src/Makefile +++ b/src/Makefile @@ -208,9 +208,9 @@ REDIS_SERVER_NAME=redis-server REDIS_SENTINEL_NAME=redis-sentinel REDIS_SERVER_OBJ=adlist.o quicklist.o ae.o anet.o dict.o server.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o cluster.o crc16.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o memtest.o crc64.o bitops.o sentinel.o notify.o setproctitle.o blocked.o hyperloglog.o latency.o sparkline.o redis-check-rdb.o redis-check-aof.o geo.o lazyfree.o module.o evict.o expire.o geohash.o geohash_helper.o childinfo.o defrag.o siphash.o rax.o t_stream.o listpack.o localtime.o lolwut.o lolwut5.o lolwut6.o acl.o gopher.o tracking.o connection.o tls.o sha256.o REDIS_CLI_NAME=redis-cli -REDIS_CLI_OBJ=anet.o adlist.o dict.o redis-cli.o zmalloc.o release.o anet.o ae.o crc64.o siphash.o crc16.o +REDIS_CLI_OBJ=anet.o adlist.o dict.o redis-cli.o zmalloc.o release.o ae.o crc64.o siphash.o crc16.o REDIS_BENCHMARK_NAME=redis-benchmark -REDIS_BENCHMARK_OBJ=ae.o anet.o redis-benchmark.o adlist.o dict.o zmalloc.o siphash.o redis-benchmark.o +REDIS_BENCHMARK_OBJ=ae.o anet.o redis-benchmark.o adlist.o dict.o zmalloc.o siphash.o REDIS_CHECK_RDB_NAME=redis-check-rdb REDIS_CHECK_AOF_NAME=redis-check-aof From c99bd27b584fb26de85465772261d87458778787 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 12 Mar 2020 12:59:44 +0100 Subject: [PATCH 0166/1280] ae.c: fix crash when resizing the event loop. See #6964. The root cause is that the event loop may be resized from an event callback itself, causing the event pointer to be invalid. --- src/ae.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/ae.c b/src/ae.c index d2248fe5c..1bf6cbfbf 100644 --- a/src/ae.c +++ b/src/ae.c @@ -464,6 +464,7 @@ int aeProcessEvents(aeEventLoop *eventLoop, int flags) if (!invert && fe->mask & mask & AE_READABLE) { fe->rfileProc(eventLoop,fd,fe->clientData,mask); fired++; + fe = &eventLoop->events[fd]; /* Refresh in case of resize. */ } /* Fire the writable event. */ @@ -476,8 +477,11 @@ int aeProcessEvents(aeEventLoop *eventLoop, int flags) /* If we have to invert the call, fire the readable event now * after the writable one. */ - if (invert && fe->mask & mask & AE_READABLE) { - if (!fired || fe->wfileProc != fe->rfileProc) { + if (invert) { + fe = &eventLoop->events[fd]; /* Refresh in case of resize. */ + if ((fe->mask & mask & AE_READABLE) && + (!fired || fe->wfileProc != fe->rfileProc)) + { fe->rfileProc(eventLoop,fd,fe->clientData,mask); fired++; } From 4278c5ad5def2d604ec9841017c6d8b882600dce Mon Sep 17 00:00:00 2001 From: fengpf <18221167541@163.com> Date: Thu, 12 Mar 2020 20:44:32 +0800 Subject: [PATCH 0167/1280] fix comments in latency.c --- .gitignore | 1 + src/latency.c | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index de626d61b..e445fd201 100644 --- a/.gitignore +++ b/.gitignore @@ -32,3 +32,4 @@ deps/lua/src/liblua.a *.dSYM Makefile.dep .vscode/* +.idea/* diff --git a/src/latency.c b/src/latency.c index 74ced72a5..9a291ac9b 100644 --- a/src/latency.c +++ b/src/latency.c @@ -85,7 +85,7 @@ int THPGetAnonHugePagesSize(void) { /* ---------------------------- Latency API --------------------------------- */ /* Latency monitor initialization. We just need to create the dictionary - * of time series, each time serie is craeted on demand in order to avoid + * of time series, each time serie is created on demand in order to avoid * having a fixed list to maintain. */ void latencyMonitorInit(void) { server.latency_events = dictCreate(&latencyTimeSeriesDictType,NULL); From ca0268c0ac127159c6fb4991a7aef2ffe702e1ef Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 12 Mar 2020 15:53:08 +0100 Subject: [PATCH 0168/1280] Update linenoise. --- deps/linenoise/linenoise.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/deps/linenoise/linenoise.c b/deps/linenoise/linenoise.c index 01c3b9350..cfe51e768 100644 --- a/deps/linenoise/linenoise.c +++ b/deps/linenoise/linenoise.c @@ -598,7 +598,8 @@ static void refreshMultiLine(struct linenoiseState *l) { /* Write the prompt and the current buffer content */ abAppend(&ab,l->prompt,strlen(l->prompt)); if (maskmode == 1) { - for (uint i = 0; i < l->len; i++) abAppend(&ab,"*",1); + unsigned int i; + for (i = 0; i < l->len; i++) abAppend(&ab,"*",1); } else { abAppend(&ab,l->buf,l->len); } From e886009f9ffae36c64a9393b52f5e4f8f0c1f91f Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 13 Mar 2020 16:21:55 +0100 Subject: [PATCH 0169/1280] Restore newline at the end of redis-cli.c --- src/redis-cli.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/redis-cli.c b/src/redis-cli.c index b44db9a1e..7ad80c0a1 100644 --- a/src/redis-cli.c +++ b/src/redis-cli.c @@ -8062,4 +8062,5 @@ int main(int argc, char **argv) { } else { return noninteractive(argc,convertToSds(argc,argv)); } -} \ No newline at end of file +} + From c022f07e04898a35a6b21c48437d4f81c734639a Mon Sep 17 00:00:00 2001 From: "zhaozhao.zz" Date: Sun, 15 Mar 2020 21:49:10 +0800 Subject: [PATCH 0170/1280] Threaded IO: bugfix #6988 process events while blocked --- src/networking.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/networking.c b/src/networking.c index 4c394af70..261c9b24a 100644 --- a/src/networking.c +++ b/src/networking.c @@ -36,6 +36,7 @@ static void setProtocolError(const char *errstr, client *c); int postponeClientRead(client *c); +int process_while_blocked; /* Return the size consumed from the allocator, for the specified SDS string, * including internal fragmentation. This function is used in order to compute @@ -2738,6 +2739,7 @@ int clientsArePaused(void) { int processEventsWhileBlocked(void) { int iterations = 4; /* See the function top-comment. */ int count = 0; + process_while_blocked = 1; while (iterations--) { int events = 0; events += aeProcessEvents(server.el, AE_FILE_EVENTS|AE_DONT_WAIT); @@ -2745,6 +2747,7 @@ int processEventsWhileBlocked(void) { if (!events) break; count += events; } + process_while_blocked = 0; return count; } @@ -2816,6 +2819,7 @@ void *IOThreadMain(void *myid) { /* Initialize the data structures needed for threaded I/O. */ void initThreadedIO(void) { io_threads_active = 0; /* We start with threads not active. */ + process_while_blocked = 0; /* Don't spawn any thread if the user selected a single thread: * we'll handle I/O directly from the main thread. */ @@ -2970,6 +2974,7 @@ int handleClientsWithPendingWritesUsingThreads(void) { int postponeClientRead(client *c) { if (io_threads_active && server.io_threads_do_reads && + !process_while_blocked && !(c->flags & (CLIENT_MASTER|CLIENT_SLAVE|CLIENT_PENDING_READ))) { c->flags |= CLIENT_PENDING_READ; From 4295673072e91ca429632cb9596b82207727baf9 Mon Sep 17 00:00:00 2001 From: antirez Date: Sun, 15 Mar 2020 16:10:37 +0100 Subject: [PATCH 0171/1280] Aesthetic changes in PR #6989. --- src/networking.c | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/networking.c b/src/networking.c index 261c9b24a..c7dd2f23c 100644 --- a/src/networking.c +++ b/src/networking.c @@ -36,7 +36,7 @@ static void setProtocolError(const char *errstr, client *c); int postponeClientRead(client *c); -int process_while_blocked; +int ProcessingEventsWhileBlocked = 0; /* See processEventsWhileBlocked(). */ /* Return the size consumed from the allocator, for the specified SDS string, * including internal fragmentation. This function is used in order to compute @@ -2739,7 +2739,12 @@ int clientsArePaused(void) { int processEventsWhileBlocked(void) { int iterations = 4; /* See the function top-comment. */ int count = 0; - process_while_blocked = 1; + + /* Note: when we are processing events while blocked (for instance during + * busy Lua scripts), we set a global flag. When such flag is set, we + * avoid handling the read part of clients using threaded I/O. + * See https://github.com/antirez/redis/issues/6988 for more info. */ + ProcessingEventsWhileBlocked = 1; while (iterations--) { int events = 0; events += aeProcessEvents(server.el, AE_FILE_EVENTS|AE_DONT_WAIT); @@ -2747,7 +2752,7 @@ int processEventsWhileBlocked(void) { if (!events) break; count += events; } - process_while_blocked = 0; + ProcessingEventsWhileBlocked = 0; return count; } @@ -2819,7 +2824,6 @@ void *IOThreadMain(void *myid) { /* Initialize the data structures needed for threaded I/O. */ void initThreadedIO(void) { io_threads_active = 0; /* We start with threads not active. */ - process_while_blocked = 0; /* Don't spawn any thread if the user selected a single thread: * we'll handle I/O directly from the main thread. */ @@ -2974,7 +2978,7 @@ int handleClientsWithPendingWritesUsingThreads(void) { int postponeClientRead(client *c) { if (io_threads_active && server.io_threads_do_reads && - !process_while_blocked && + !ProcessingEventsWhileBlocked && !(c->flags & (CLIENT_MASTER|CLIENT_SLAVE|CLIENT_PENDING_READ))) { c->flags |= CLIENT_PENDING_READ; From 078c97ea4403b27a96ea682e338505301f0e0dd8 Mon Sep 17 00:00:00 2001 From: "zhaozhao.zz" Date: Sun, 15 Mar 2020 23:30:25 +0800 Subject: [PATCH 0172/1280] Threaded IO: bugfix client kill may crash redis --- src/networking.c | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/src/networking.c b/src/networking.c index 261c9b24a..b6ae9bd50 100644 --- a/src/networking.c +++ b/src/networking.c @@ -3036,16 +3036,22 @@ int handleClientsWithPendingReadsUsingThreads(void) { if (tio_debug) printf("I/O READ All threads finshed\n"); /* Run the list of clients again to process the new buffers. */ - listRewind(server.clients_pending_read,&li); - while((ln = listNext(&li))) { + while(listLength(server.clients_pending_read)) { + ln = listFirst(server.clients_pending_read); client *c = listNodeValue(ln); c->flags &= ~CLIENT_PENDING_READ; + listDelNode(server.clients_pending_read,ln); + if (c->flags & CLIENT_PENDING_COMMAND) { - c->flags &= ~ CLIENT_PENDING_COMMAND; - processCommandAndResetClient(c); + c->flags &= ~CLIENT_PENDING_COMMAND; + if (processCommandAndResetClient(c) == C_ERR) { + /* If the client is no longer valid, we avoid + * processing the client later. So we just go + * to the next. */ + continue; + } } processInputBufferAndReplicate(c); } - listEmpty(server.clients_pending_read); return processed; } From a02e9fc81d9924af00daac8e844ce59fc196edc5 Mon Sep 17 00:00:00 2001 From: "zhaozhao.zz" Date: Mon, 16 Mar 2020 11:20:48 +0800 Subject: [PATCH 0173/1280] Threaded IO: handle pending reads clients ASAP after event loop --- src/server.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/server.c b/src/server.c index a6d4b357e..f702da94a 100644 --- a/src/server.c +++ b/src/server.c @@ -2088,6 +2088,9 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) { void beforeSleep(struct aeEventLoop *eventLoop) { UNUSED(eventLoop); + /* We should handle pending reads clients ASAP after event loop. */ + handleClientsWithPendingReadsUsingThreads(); + /* Handle TLS pending data. (must be done before flushAppendOnlyFile) */ tlsProcessPendingData(); /* If tls still has pending unread data don't sleep at all. */ @@ -2157,7 +2160,6 @@ void beforeSleep(struct aeEventLoop *eventLoop) { void afterSleep(struct aeEventLoop *eventLoop) { UNUSED(eventLoop); if (moduleCount()) moduleAcquireGIL(); - handleClientsWithPendingReadsUsingThreads(); } /* =========================== Server initialization ======================== */ From bd35171848dc89508830502f30fea94867164071 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 16 Mar 2020 13:48:29 +0100 Subject: [PATCH 0174/1280] Example sentinel conf: document requirepass. --- sentinel.conf | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/sentinel.conf b/sentinel.conf index bc9a705ac..796f45088 100644 --- a/sentinel.conf +++ b/sentinel.conf @@ -112,6 +112,14 @@ sentinel monitor mymaster 127.0.0.1 6379 2 # Default is 30 seconds. sentinel down-after-milliseconds mymaster 30000 +# requirepass +# +# You can configure Sentinel itself to require a password, however when doing +# so Sentinel will try to authenticate with the same password to all the +# other Sentinels. So you need to configure all your Sentinels in a given +# group with the same "requirepass" password. Check the following documentation +# for more info: https://redis.io/topics/sentinel + # sentinel parallel-syncs # # How many replicas we can reconfigure to point to the new replica simultaneously From 411aadd83e680954f638de210827c73088c002e1 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 16 Mar 2020 15:59:29 +0100 Subject: [PATCH 0175/1280] Sentinel: implement auth-user directive for ACLs. --- src/sentinel.c | 45 ++++++++++++++++++++++++++++++++++++++------- 1 file changed, 38 insertions(+), 7 deletions(-) diff --git a/src/sentinel.c b/src/sentinel.c index 10c003d03..83d6c00bb 100644 --- a/src/sentinel.c +++ b/src/sentinel.c @@ -205,7 +205,8 @@ typedef struct sentinelRedisInstance { dict *slaves; /* Slaves for this master instance. */ unsigned int quorum;/* Number of sentinels that need to agree on failure. */ int parallel_syncs; /* How many slaves to reconfigure at same time. */ - char *auth_pass; /* Password to use for AUTH against master & slaves. */ + char *auth_pass; /* Password to use for AUTH against master & replica. */ + char *auth_user; /* Username for ACLs AUTH against master & replica. */ /* Slave specific. */ mstime_t master_link_down_time; /* Slave replication link down time. */ @@ -1231,6 +1232,7 @@ sentinelRedisInstance *createSentinelRedisInstance(char *name, int flags, char * SENTINEL_DEFAULT_DOWN_AFTER; ri->master_link_down_time = 0; ri->auth_pass = NULL; + ri->auth_user = NULL; ri->slave_priority = SENTINEL_DEFAULT_SLAVE_PRIORITY; ri->slave_reconf_sent_time = 0; ri->slave_master_host = NULL; @@ -1289,6 +1291,7 @@ void releaseSentinelRedisInstance(sentinelRedisInstance *ri) { sdsfree(ri->slave_master_host); sdsfree(ri->leader); sdsfree(ri->auth_pass); + sdsfree(ri->auth_user); sdsfree(ri->info); releaseSentinelAddr(ri->addr); dictRelease(ri->renamed_commands); @@ -1654,19 +1657,19 @@ char *sentinelHandleConfiguration(char **argv, int argc) { ri->failover_timeout = atoi(argv[2]); if (ri->failover_timeout <= 0) return "negative or zero time parameter."; - } else if (!strcasecmp(argv[0],"parallel-syncs") && argc == 3) { + } else if (!strcasecmp(argv[0],"parallel-syncs") && argc == 3) { /* parallel-syncs */ ri = sentinelGetMasterByName(argv[1]); if (!ri) return "No such master with specified name."; ri->parallel_syncs = atoi(argv[2]); - } else if (!strcasecmp(argv[0],"notification-script") && argc == 3) { + } else if (!strcasecmp(argv[0],"notification-script") && argc == 3) { /* notification-script */ ri = sentinelGetMasterByName(argv[1]); if (!ri) return "No such master with specified name."; if (access(argv[2],X_OK) == -1) return "Notification script seems non existing or non executable."; ri->notification_script = sdsnew(argv[2]); - } else if (!strcasecmp(argv[0],"client-reconfig-script") && argc == 3) { + } else if (!strcasecmp(argv[0],"client-reconfig-script") && argc == 3) { /* client-reconfig-script */ ri = sentinelGetMasterByName(argv[1]); if (!ri) return "No such master with specified name."; @@ -1674,11 +1677,16 @@ char *sentinelHandleConfiguration(char **argv, int argc) { return "Client reconfiguration script seems non existing or " "non executable."; ri->client_reconfig_script = sdsnew(argv[2]); - } else if (!strcasecmp(argv[0],"auth-pass") && argc == 3) { + } else if (!strcasecmp(argv[0],"auth-pass") && argc == 3) { /* auth-pass */ ri = sentinelGetMasterByName(argv[1]); if (!ri) return "No such master with specified name."; ri->auth_pass = sdsnew(argv[2]); + } else if (!strcasecmp(argv[0],"auth-user") && argc == 3) { + /* auth-user */ + ri = sentinelGetMasterByName(argv[1]); + if (!ri) return "No such master with specified name."; + ri->auth_user = sdsnew(argv[2]); } else if (!strcasecmp(argv[0],"current-epoch") && argc == 2) { /* current-epoch */ unsigned long long current_epoch = strtoull(argv[1],NULL,10); @@ -1836,7 +1844,7 @@ void rewriteConfigSentinelOption(struct rewriteConfigState *state) { rewriteConfigRewriteLine(state,"sentinel",line,1); } - /* sentinel auth-pass */ + /* sentinel auth-pass & auth-user */ if (master->auth_pass) { line = sdscatprintf(sdsempty(), "sentinel auth-pass %s %s", @@ -1844,6 +1852,13 @@ void rewriteConfigSentinelOption(struct rewriteConfigState *state) { rewriteConfigRewriteLine(state,"sentinel",line,1); } + if (master->auth_user) { + line = sdscatprintf(sdsempty(), + "sentinel auth-user %s %s", + master->name, master->auth_user); + rewriteConfigRewriteLine(state,"sentinel",line,1); + } + /* sentinel config-epoch */ line = sdscatprintf(sdsempty(), "sentinel config-epoch %s %llu", @@ -1968,19 +1983,29 @@ werr: * will disconnect and reconnect the link and so forth. */ void sentinelSendAuthIfNeeded(sentinelRedisInstance *ri, redisAsyncContext *c) { char *auth_pass = NULL; + char *auth_user = NULL; if (ri->flags & SRI_MASTER) { auth_pass = ri->auth_pass; + auth_user = ri->auth_user; } else if (ri->flags & SRI_SLAVE) { auth_pass = ri->master->auth_pass; + auth_user = ri->master->auth_user; } else if (ri->flags & SRI_SENTINEL) { auth_pass = ACLDefaultUserFirstPassword(); + auth_user = NULL; } - if (auth_pass) { + if (auth_pass && auth_user == NULL) { if (redisAsyncCommand(c, sentinelDiscardReplyCallback, ri, "%s %s", sentinelInstanceMapCommand(ri,"AUTH"), auth_pass) == C_OK) ri->link->pending_commands++; + } else if (auth_pass && auth_user) { + /* If we also have an username, use the ACL-style AUTH command + * with two arguments, username and password. */ + if (redisAsyncCommand(c, sentinelDiscardReplyCallback, ri, "%s %s %s", + sentinelInstanceMapCommand(ri,"AUTH"), + auth_user, auth_pass) == C_OK) ri->link->pending_commands++; } } @@ -3522,6 +3547,12 @@ void sentinelSetCommand(client *c) { sdsfree(ri->auth_pass); ri->auth_pass = strlen(value) ? sdsnew(value) : NULL; changes++; + } else if (!strcasecmp(option,"auth-user") && moreargs > 0) { + /* auth-user */ + char *value = c->argv[++j]->ptr; + sdsfree(ri->auth_user); + ri->auth_user = strlen(value) ? sdsnew(value) : NULL; + changes++; } else if (!strcasecmp(option,"quorum") && moreargs > 0) { /* quorum */ robj *o = c->argv[++j]; From 0de01d8e514e68bb9eb4511ace1cbd0027913717 Mon Sep 17 00:00:00 2001 From: artix Date: Tue, 18 Feb 2020 10:46:10 +0100 Subject: [PATCH 0176/1280] Support Redis Cluster Proxy PROXY INFO command --- src/redis-cli.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/redis-cli.c b/src/redis-cli.c index 7ad80c0a1..7e440d67c 100644 --- a/src/redis-cli.c +++ b/src/redis-cli.c @@ -1292,7 +1292,11 @@ static int cliSendCommand(int argc, char **argv, long repeat) { (argc == 3 && !strcasecmp(command,"latency") && !strcasecmp(argv[1],"graph")) || (argc == 2 && !strcasecmp(command,"latency") && - !strcasecmp(argv[1],"doctor"))) + !strcasecmp(argv[1],"doctor")) || + /* Format PROXY INFO command for Redis Cluster Proxy: + * https://github.com/artix75/redis-cluster-proxy */ + (argc >= 2 && !strcasecmp(command,"proxy") && + !strcasecmp(argv[1],"info"))) { output_raw = 1; } From 3b5ca2f19fc1165cb7a98eaf4375f46babd5569f Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 16 Mar 2020 16:56:50 +0100 Subject: [PATCH 0177/1280] ACL: Make Redis 6 more backward compatible with requirepass. Note that this as a side effect fixes Sentinel "requirepass" mode. --- src/acl.c | 11 +---------- src/config.c | 16 ++++++++++++---- src/sentinel.c | 2 +- src/server.h | 3 +++ 4 files changed, 17 insertions(+), 15 deletions(-) diff --git a/src/acl.c b/src/acl.c index efe6b96ad..27f4bdb84 100644 --- a/src/acl.c +++ b/src/acl.c @@ -899,16 +899,6 @@ char *ACLSetUserStringError(void) { return errmsg; } -/* Return the first password of the default user or NULL. - * This function is needed for backward compatibility with the old - * directive "requirepass" when Redis supported a single global - * password. */ -sds ACLDefaultUserFirstPassword(void) { - if (listLength(DefaultUser->passwords) == 0) return NULL; - listNode *first = listFirst(DefaultUser->passwords); - return listNodeValue(first); -} - /* Initialize the default user, that will always exist for all the process * lifetime. */ void ACLInitDefaultUser(void) { @@ -925,6 +915,7 @@ void ACLInit(void) { UsersToLoad = listCreate(); ACLLog = listCreate(); ACLInitDefaultUser(); + server.requirepass = NULL; /* Only used for backward compatibility. */ } /* Check the username and password pair and return C_OK if they are valid, diff --git a/src/config.c b/src/config.c index 211b6d003..7c87ebe6e 100644 --- a/src/config.c +++ b/src/config.c @@ -411,11 +411,15 @@ void loadServerConfigFromString(char *config) { goto loaderr; } /* The old "requirepass" directive just translates to setting - * a password to the default user. */ + * a password to the default user. The only thing we do + * additionally is to remember the cleartext password in this + * case, for backward compatibility with Redis <= 5. */ ACLSetUser(DefaultUser,"resetpass",-1); sds aclop = sdscatprintf(sdsempty(),">%s",argv[1]); ACLSetUser(DefaultUser,aclop,sdslen(aclop)); sdsfree(aclop); + sdsfree(server.requirepass); + server.requirepass = sdsnew(argv[1]); } else if (!strcasecmp(argv[0],"list-max-ziplist-entries") && argc == 2){ /* DEAD OPTION */ } else if (!strcasecmp(argv[0],"list-max-ziplist-value") && argc == 2) { @@ -623,11 +627,15 @@ void configSetCommand(client *c) { config_set_special_field("requirepass") { if (sdslen(o->ptr) > CONFIG_AUTHPASS_MAX_LEN) goto badfmt; /* The old "requirepass" directive just translates to setting - * a password to the default user. */ + * a password to the default user. The only thing we do + * additionally is to remember the cleartext password in this + * case, for backward compatibility with Redis <= 5. */ ACLSetUser(DefaultUser,"resetpass",-1); sds aclop = sdscatprintf(sdsempty(),">%s",(char*)o->ptr); ACLSetUser(DefaultUser,aclop,sdslen(aclop)); sdsfree(aclop); + sdsfree(server.requirepass); + server.requirepass = sdsnew(o->ptr); } config_set_special_field("save") { int vlen, j; sds *v = sdssplitlen(o->ptr,sdslen(o->ptr)," ",1,&vlen); @@ -899,7 +907,7 @@ void configGetCommand(client *c) { } if (stringmatch(pattern,"requirepass",1)) { addReplyBulkCString(c,"requirepass"); - sds password = ACLDefaultUserFirstPassword(); + sds password = server.requirepass; if (password) { addReplyBulkCBuffer(c,password,sdslen(password)); } else { @@ -1341,7 +1349,7 @@ void rewriteConfigBindOption(struct rewriteConfigState *state) { void rewriteConfigRequirepassOption(struct rewriteConfigState *state, char *option) { int force = 1; sds line; - sds password = ACLDefaultUserFirstPassword(); + sds password = server.requirepass; /* If there is no password set, we don't want the requirepass option * to be present in the configuration at all. */ diff --git a/src/sentinel.c b/src/sentinel.c index 83d6c00bb..d091bf230 100644 --- a/src/sentinel.c +++ b/src/sentinel.c @@ -1992,7 +1992,7 @@ void sentinelSendAuthIfNeeded(sentinelRedisInstance *ri, redisAsyncContext *c) { auth_pass = ri->master->auth_pass; auth_user = ri->master->auth_user; } else if (ri->flags & SRI_SENTINEL) { - auth_pass = ACLDefaultUserFirstPassword(); + auth_pass = server.requirepass; auth_user = NULL; } diff --git a/src/server.h b/src/server.h index 3c19a17ea..fa6770dfa 100644 --- a/src/server.h +++ b/src/server.h @@ -1395,6 +1395,9 @@ struct redisServer { /* ACLs */ char *acl_filename; /* ACL Users file. NULL if not configured. */ unsigned long acllog_max_len; /* Maximum length of the ACL LOG list. */ + sds requirepass; /* Remember the cleartext password set with the + old "requirepass" directive for backward + compatibility with Redis <= 5. */ /* Assert & bug reporting */ const char *assert_failed; const char *assert_file; From 498e28f98b51237277605e926bfaebd54b543df6 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 16 Mar 2020 17:11:43 +0100 Subject: [PATCH 0178/1280] Sentinel: document auth-user directive. --- sentinel.conf | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/sentinel.conf b/sentinel.conf index 796f45088..4ca5e5f8f 100644 --- a/sentinel.conf +++ b/sentinel.conf @@ -102,6 +102,18 @@ sentinel monitor mymaster 127.0.0.1 6379 2 # # sentinel auth-pass mymaster MySUPER--secret-0123passw0rd +# sentinel auth-user +# +# This is useful in order to authenticate to instances having ACL capabilities, +# that is, running Redis 6.0 or greater. When just auth-pass is provided the +# Sentinel instance will authenticate to Redis using the old "AUTH " +# method. When also an username is provided, it will use "AUTH ". +# In the Redis servers side, the ACL to provide just minimal access to +# Sentinel instances, should be configured along the following lines: +# +# user sentinel-user >somepassword +client +subscribe +publish \ +# +ping +info +multi +slaveof +config +client +exec on + # sentinel down-after-milliseconds # # Number of milliseconds the master (or any attached replica or sentinel) should From 70705e758dc6ecf887dd472de98ff72d8a60fc48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EB=B0=95=EC=8A=B9=ED=98=84?= Date: Wed, 18 Mar 2020 14:40:50 +0900 Subject: [PATCH 0179/1280] Update redis.conf --- redis.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/redis.conf b/redis.conf index c9d256bef..7c55a3ab0 100644 --- a/redis.conf +++ b/redis.conf @@ -1628,7 +1628,7 @@ hz 10 # offers, and enables by default, the ability to use an adaptive HZ value # which will temporary raise when there are many connected clients. # -# When dynamic HZ is enabled, the actual configured HZ will be used as +# When dynamic HZ is enabled, the actual configured HZ will be used # as a baseline, but multiples of the configured HZ value will be actually # used as needed once more clients are connected. In this way an idle # instance will use very little CPU time while a busy instance will be From 0c9916d008247df5cef12180495d9221eee6f60d Mon Sep 17 00:00:00 2001 From: WuYunlong Date: Wed, 18 Mar 2020 16:17:46 +0800 Subject: [PATCH 0180/1280] Add 14-consistency-check.tcl to prove there is a data consistency issue. --- tests/cluster/tests/14-consistency-check.tcl | 87 ++++++++++++++++++++ 1 file changed, 87 insertions(+) create mode 100644 tests/cluster/tests/14-consistency-check.tcl diff --git a/tests/cluster/tests/14-consistency-check.tcl b/tests/cluster/tests/14-consistency-check.tcl new file mode 100644 index 000000000..a43725ebc --- /dev/null +++ b/tests/cluster/tests/14-consistency-check.tcl @@ -0,0 +1,87 @@ +source "../tests/includes/init-tests.tcl" + +test "Create a 5 nodes cluster" { + create_cluster 5 5 +} + +test "Cluster should start ok" { + assert_cluster_state ok +} + +test "Cluster is writable" { + cluster_write_test 0 +} + +proc find_non_empty_master {} { + set master_id_no {} + foreach_redis_id id { + if {[RI $id role] eq {master} && [R $id dbsize] > 0} { + set master_id_no $id + } + } + return $master_id_no +} + +proc get_one_of_my_replica {id} { + set replica_port [lindex [lindex [lindex [R $id role] 2] 0] 1] + set replica_id_num [get_instance_id_by_port redis $replica_port] + return $replica_id_num +} + +proc cluster_write_keys_with_expire {id ttl} { + set prefix [randstring 20 20 alpha] + set port [get_instance_attrib redis $id port] + set cluster [redis_cluster 127.0.0.1:$port] + for {set j 100} {$j < 200} {incr j} { + $cluster setex key_expire.$j $ttl $prefix.$j + } + $cluster close +} + +proc test_slave_load_expired_keys {aof} { + test "Slave expired keys is loaded when restarted: appendonly=$aof" { + set master_id [find_non_empty_master] + set replica_id [get_one_of_my_replica $master_id] + + set master_dbsize [R $master_id dbsize] + set slave_dbsize [R $replica_id dbsize] + assert_equal $master_dbsize $slave_dbsize + + set data_ttl 5 + cluster_write_keys_with_expire $master_id $data_ttl + after 100 + set replica_dbsize_1 [R $replica_id dbsize] + assert {$replica_dbsize_1 > $slave_dbsize} + + R $replica_id config set appendonly $aof + R $replica_id config rewrite + + set start_time [clock seconds] + set end_time [expr $start_time+$data_ttl+2] + R $replica_id save + set replica_dbsize_2 [R $replica_id dbsize] + assert {$replica_dbsize_2 > $slave_dbsize} + kill_instance redis $replica_id + + set master_port [get_instance_attrib redis $master_id port] + exec ../../../src/redis-cli -h 127.0.0.1 -p $master_port debug sleep [expr $data_ttl+3] > /dev/null & + + while {[clock seconds] <= $end_time} { + #wait for $data_ttl seconds + } + restart_instance redis $replica_id + + wait_for_condition 200 50 { + [R $replica_id ping] eq {PONG} + } else { + fail "replica #$replica_id not started" + } + + set replica_dbsize_3 [R $replica_id dbsize] + assert {$replica_dbsize_3 > $slave_dbsize} + } +} + +test_slave_load_expired_keys no +after 5000 +test_slave_load_expired_keys yes From f4a6b931cceb33ce730cc1ba4bb3c09070d275d3 Mon Sep 17 00:00:00 2001 From: WuYunlong Date: Wed, 18 Mar 2020 16:20:10 +0800 Subject: [PATCH 0181/1280] Fix master replica inconsistency for upgrading scenario. Before this commit, when upgrading a replica, expired keys will not be loaded, thus causing replica having less keys in db. To this point, master and replica's keys is logically consistent. However, before the keys in master and replica are physically consistent, that is, they have the same dbsize, if master got a problem and the replica got promoted and becomes new master of that partition, and master updates a key which does not exist on master, but physically exists on the old master(new replica), the old master would refuse to update the key, thus causing master and replica data inconsistent. How could this happen? That's all because of the wrong judgement of roles while starting up the server. We can not use server.masterhost to judge if the server is master or replica, since it fails in cluster mode. When we start the server, we load rdb and do want to load expired keys, and do not want to have the ability to active expire keys, if it is a replica. --- src/rdb.c | 2 +- src/server.c | 7 ++++++- src/server.h | 2 ++ 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/rdb.c b/src/rdb.c index cbcea96c6..5d34f5a32 100644 --- a/src/rdb.c +++ b/src/rdb.c @@ -2231,7 +2231,7 @@ int rdbLoadRio(rio *rdb, int rdbflags, rdbSaveInfo *rsi) { * received from the master. In the latter case, the master is * responsible for key expiry. If we would expire keys here, the * snapshot taken by the master may not be reflected on the slave. */ - if (server.masterhost == NULL && !(rdbflags&RDBFLAGS_AOF_PREAMBLE) && expiretime != -1 && expiretime < now) { + if (iAmMaster() && !(rdbflags&RDBFLAGS_AOF_PREAMBLE) && expiretime != -1 && expiretime < now) { decrRefCount(key); decrRefCount(val); } else { diff --git a/src/server.c b/src/server.c index f702da94a..467d09b67 100644 --- a/src/server.c +++ b/src/server.c @@ -1691,7 +1691,7 @@ void databasesCron(void) { /* Expire keys by random sampling. Not required for slaves * as master will synthesize DELs for us. */ if (server.active_expire_enabled) { - if (server.masterhost == NULL) { + if (iAmMaster()) { activeExpireCycle(ACTIVE_EXPIRE_CYCLE_SLOW); } else { expireSlaveKeys(); @@ -4863,6 +4863,11 @@ int redisIsSupervised(int mode) { return 0; } +int iAmMaster(void) { + return ((!server.cluster_enabled && server.masterhost == NULL) || + (server.cluster_enabled && nodeIsMaster(server.cluster->myself))); +} + int main(int argc, char **argv) { struct timeval tv; diff --git a/src/server.h b/src/server.h index fa6770dfa..fdfe5b8ea 100644 --- a/src/server.h +++ b/src/server.h @@ -2393,4 +2393,6 @@ int tlsConfigure(redisTLSContextConfig *ctx_config); #define redisDebugMark() \ printf("-- MARK %s:%d --\n", __FILE__, __LINE__) +int iAmMaster(void); + #endif From 44c3c39dde77bbaf461ecea7f6edaa326d73edf0 Mon Sep 17 00:00:00 2001 From: Guy Benoish Date: Wed, 18 Mar 2020 18:34:27 +0530 Subject: [PATCH 0182/1280] Allow RM_GetContextFlags to work with ctx==NULL --- src/module.c | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/src/module.c b/src/module.c index bbd54082c..a5f4219fd 100644 --- a/src/module.c +++ b/src/module.c @@ -1848,20 +1848,22 @@ int RM_GetContextFlags(RedisModuleCtx *ctx) { int flags = 0; /* Client specific flags */ - if (ctx->client) { - if (ctx->client->flags & CLIENT_LUA) - flags |= REDISMODULE_CTX_FLAGS_LUA; - if (ctx->client->flags & CLIENT_MULTI) - flags |= REDISMODULE_CTX_FLAGS_MULTI; - /* Module command recieved from MASTER, is replicated. */ - if (ctx->client->flags & CLIENT_MASTER) - flags |= REDISMODULE_CTX_FLAGS_REPLICATED; - } + if (ctx) { + if (ctx->client) { + if (ctx->client->flags & CLIENT_LUA) + flags |= REDISMODULE_CTX_FLAGS_LUA; + if (ctx->client->flags & CLIENT_MULTI) + flags |= REDISMODULE_CTX_FLAGS_MULTI; + /* Module command recieved from MASTER, is replicated. */ + if (ctx->client->flags & CLIENT_MASTER) + flags |= REDISMODULE_CTX_FLAGS_REPLICATED; + } - /* For DIRTY flags, we need the blocked client if used */ - client *c = ctx->blocked_client ? ctx->blocked_client->client : ctx->client; - if (c && (c->flags & (CLIENT_DIRTY_CAS|CLIENT_DIRTY_EXEC))) { - flags |= REDISMODULE_CTX_FLAGS_MULTI_DIRTY; + /* For DIRTY flags, we need the blocked client if used */ + client *c = ctx->blocked_client ? ctx->blocked_client->client : ctx->client; + if (c && (c->flags & (CLIENT_DIRTY_CAS|CLIENT_DIRTY_EXEC))) { + flags |= REDISMODULE_CTX_FLAGS_MULTI_DIRTY; + } } if (server.cluster_enabled) From ee1443973cbf9b741dc956d04d9b18e11f45db7e Mon Sep 17 00:00:00 2001 From: hwware Date: Wed, 18 Mar 2020 09:33:52 -0400 Subject: [PATCH 0183/1280] fix potentical memory leak in redis-cli --- src/redis-cli.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/redis-cli.c b/src/redis-cli.c index 7ad80c0a1..c0cc69592 100644 --- a/src/redis-cli.c +++ b/src/redis-cli.c @@ -1989,6 +1989,8 @@ static void repl(void) { if (config.eval) { config.eval_ldb = 1; config.output = OUTPUT_RAW; + sdsfreesplitres(argv,argc); + linenoiseFree(line); return; /* Return to evalMode to restart the session. */ } else { printf("Use 'restart' only in Lua debugging mode."); From d59e54c32cf66a370827d315f7aaa7ec43c3de38 Mon Sep 17 00:00:00 2001 From: hwware Date: Wed, 18 Mar 2020 09:48:03 -0400 Subject: [PATCH 0184/1280] fix spelling in cluster.c --- src/cluster.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cluster.c b/src/cluster.c index 5f63d2b8f..f67e97abd 100644 --- a/src/cluster.c +++ b/src/cluster.c @@ -932,7 +932,7 @@ int clusterAddNode(clusterNode *node) { return (retval == DICT_OK) ? C_OK : C_ERR; } -/* Remove a node from the cluster. The functio performs the high level +/* Remove a node from the cluster. The function performs the high level * cleanup, calling freeClusterNode() for the low level cleanup. * Here we do the following: * From 90f5d8338154b5c2b9227f5f752a463f3955ab98 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 20 Mar 2020 12:45:48 +0100 Subject: [PATCH 0185/1280] ACL: default user off should not allow automatic authentication. This fixes issue #7011. --- src/networking.c | 3 ++- src/server.c | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/networking.c b/src/networking.c index 0690bbdf6..69d59a59b 100644 --- a/src/networking.c +++ b/src/networking.c @@ -124,7 +124,8 @@ client *createClient(connection *conn) { c->ctime = c->lastinteraction = server.unixtime; /* If the default user does not require authentication, the user is * directly authenticated. */ - c->authenticated = (c->user->flags & USER_FLAG_NOPASS) != 0; + c->authenticated = (c->user->flags & USER_FLAG_NOPASS) && + !(c->user->flags & USER_FLAG_DISABLED); c->replstate = REPL_STATE_NONE; c->repl_put_online_on_ack = 0; c->reploff = 0; diff --git a/src/server.c b/src/server.c index f702da94a..612805ce5 100644 --- a/src/server.c +++ b/src/server.c @@ -3380,7 +3380,7 @@ int processCommand(client *c) { /* Check if the user is authenticated. This check is skipped in case * the default user is flagged as "nopass" and is active. */ int auth_required = (!(DefaultUser->flags & USER_FLAG_NOPASS) || - DefaultUser->flags & USER_FLAG_DISABLED) && + (DefaultUser->flags & USER_FLAG_DISABLED)) && !c->authenticated; if (auth_required) { /* AUTH and HELLO and no auth modules are valid even in From bbbd16a90d0fdcdbb73a84b1b6ffe4c45864f6ec Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 20 Mar 2020 12:52:06 +0100 Subject: [PATCH 0186/1280] Regression test for #7011. --- tests/unit/acl.tcl | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/unit/acl.tcl b/tests/unit/acl.tcl index fc1664a75..85c9b81a9 100644 --- a/tests/unit/acl.tcl +++ b/tests/unit/acl.tcl @@ -248,4 +248,11 @@ start_server {tags {"acl"}} { r AUTH default "" assert {[llength [r ACL LOG]] == 5} } + + test {When default user is off, new connections are not authenticated} { + r ACL setuser default off + catch {set rd1 [redis_deferring_client]} e + r ACL setuser default on + set e + } {*NOAUTH*} } From 8d8644b330a2ff02bcba5e498ec95732acfa648c Mon Sep 17 00:00:00 2001 From: hwware Date: Fri, 20 Mar 2020 02:40:54 -0400 Subject: [PATCH 0187/1280] add missing commands in cluster help --- src/cluster.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/cluster.c b/src/cluster.c index 5f63d2b8f..72755823a 100644 --- a/src/cluster.c +++ b/src/cluster.c @@ -4261,7 +4261,7 @@ void clusterCommand(client *c) { "FORGET -- Remove a node from the cluster.", "GETKEYSINSLOT -- Return key names stored by current node in a slot.", "FLUSHSLOTS -- Delete current node own slots information.", -"INFO - Return onformation about the cluster.", +"INFO - Return information about the cluster.", "KEYSLOT -- Return the hash slot for .", "MEET [bus-port] -- Connect nodes into a working cluster.", "MYID -- Return the node id.", @@ -4272,6 +4272,7 @@ void clusterCommand(client *c) { "SET-config-epoch - Set config epoch of current node.", "SETSLOT (importing|migrating|stable|node ) -- Set slot state.", "REPLICAS -- Return replicas.", +"SAVECONFIG - Force saving cluster configuration on disk.", "SLOTS -- Return information about slots range mappings. Each range is made of:", " start, end, master and replicas IP addresses, ports and ids", NULL From efea1b200156a66e27ec1d029457efed76fc63fd Mon Sep 17 00:00:00 2001 From: Yossi Gottlieb Date: Sun, 22 Mar 2020 14:42:03 +0200 Subject: [PATCH 0188/1280] Conns: Fix connClose() / connAccept() behavior. We assume accept handlers may choose to reject a connection and close it, but connAccept() callers can't distinguish between this state and other error states requiring connClose(). This makes it safe (and mandatory!) to always call connClose() if connAccept() fails, and safe for accept handlers to close connections (which will defer). --- src/connection.c | 12 ++++++++--- src/connection.h | 15 ++++++++++---- src/connhelpers.h | 53 +++++++++++++++++++++++++---------------------- 3 files changed, 48 insertions(+), 32 deletions(-) diff --git a/src/connection.c b/src/connection.c index 58d86c31b..2015c9195 100644 --- a/src/connection.c +++ b/src/connection.c @@ -152,7 +152,7 @@ static void connSocketClose(connection *conn) { /* If called from within a handler, schedule the close but * keep the connection until the handler returns. */ - if (conn->flags & CONN_FLAG_IN_HANDLER) { + if (connHasRefs(conn)) { conn->flags |= CONN_FLAG_CLOSE_SCHEDULED; return; } @@ -183,10 +183,16 @@ static int connSocketRead(connection *conn, void *buf, size_t buf_len) { } static int connSocketAccept(connection *conn, ConnectionCallbackFunc accept_handler) { + int ret = C_OK; + if (conn->state != CONN_STATE_ACCEPTING) return C_ERR; conn->state = CONN_STATE_CONNECTED; - if (!callHandler(conn, accept_handler)) return C_ERR; - return C_OK; + + connIncrRefs(conn); + if (!callHandler(conn, accept_handler)) ret = C_ERR; + connDecrRefs(conn); + + return ret; } /* Register a write handler, to be called when the connection is writable. diff --git a/src/connection.h b/src/connection.h index 97622f8d6..db09dfd83 100644 --- a/src/connection.h +++ b/src/connection.h @@ -45,9 +45,8 @@ typedef enum { CONN_STATE_ERROR } ConnectionState; -#define CONN_FLAG_IN_HANDLER (1<<0) /* A handler execution is in progress */ -#define CONN_FLAG_CLOSE_SCHEDULED (1<<1) /* Closed scheduled by a handler */ -#define CONN_FLAG_WRITE_BARRIER (1<<2) /* Write barrier requested */ +#define CONN_FLAG_CLOSE_SCHEDULED (1<<0) /* Closed scheduled by a handler */ +#define CONN_FLAG_WRITE_BARRIER (1<<1) /* Write barrier requested */ typedef void (*ConnectionCallbackFunc)(struct connection *conn); @@ -70,7 +69,8 @@ typedef struct ConnectionType { struct connection { ConnectionType *type; ConnectionState state; - int flags; + short int flags; + short int refs; int last_errno; void *private_data; ConnectionCallbackFunc conn_handler; @@ -88,6 +88,13 @@ struct connection { * connAccept() may directly call accept_handler(), or return and call it * at a later time. This behavior is a bit awkward but aims to reduce the need * to wait for the next event loop, if no additional handshake is required. + * + * IMPORTANT: accept_handler may decide to close the connection, calling connClose(). + * To make this safe, the connection is only marked with CONN_FLAG_CLOSE_SCHEDULED + * in this case, and connAccept() returns with an error. + * + * connAccept() callers must always check the return value and on error (C_ERR) + * a connClose() must be called. */ static inline int connAccept(connection *conn, ConnectionCallbackFunc accept_handler) { diff --git a/src/connhelpers.h b/src/connhelpers.h index f237c9b1d..86250d09e 100644 --- a/src/connhelpers.h +++ b/src/connhelpers.h @@ -37,46 +37,49 @@ * implementations (currently sockets in connection.c and TLS in tls.c). * * Currently helpers implement the mechanisms for invoking connection - * handlers, tracking in-handler states and dealing with deferred - * destruction (if invoked by a handler). + * handlers and tracking connection references, to allow safe destruction + * of connections from within a handler. */ -/* Called whenever a handler is invoked on a connection and sets the - * CONN_FLAG_IN_HANDLER flag to indicate we're in a handler context. +/* Incremenet connection references. * - * An attempt to close a connection while CONN_FLAG_IN_HANDLER is - * set will result with deferred close, i.e. setting the CONN_FLAG_CLOSE_SCHEDULED - * instead of destructing it. + * Inside a connection handler, we guarantee refs >= 1 so it is always + * safe to connClose(). + * + * In other cases where we don't want to prematurely lose the connection, + * it can go beyond 1 as well; currently it is only done by connAccept(). */ -static inline void enterHandler(connection *conn) { - conn->flags |= CONN_FLAG_IN_HANDLER; +static inline void connIncrRefs(connection *conn) { + conn->refs++; } -/* Called whenever a handler returns. This unsets the CONN_FLAG_IN_HANDLER - * flag and performs actual close/destruction if a deferred close was - * scheduled by the handler. +/* Decrement connection references. + * + * Note that this is not intended to provide any automatic free logic! + * callHandler() takes care of that for the common flows, and anywhere an + * explicit connIncrRefs() is used, the caller is expected to take care of + * that. */ -static inline int exitHandler(connection *conn) { - conn->flags &= ~CONN_FLAG_IN_HANDLER; - if (conn->flags & CONN_FLAG_CLOSE_SCHEDULED) { - connClose(conn); - return 0; - } - return 1; + +static inline void connDecrRefs(connection *conn) { + conn->refs--; +} + +static inline int connHasRefs(connection *conn) { + return conn->refs; } /* Helper for connection implementations to call handlers: - * 1. Mark the handler in use. + * 1. Increment refs to protect the connection. * 2. Execute the handler (if set). - * 3. Mark the handler as NOT in use and perform deferred close if was - * requested by the handler at any time. + * 3. Decrement refs and perform deferred close, if refs==0. */ static inline int callHandler(connection *conn, ConnectionCallbackFunc handler) { - conn->flags |= CONN_FLAG_IN_HANDLER; + connIncrRefs(conn); if (handler) handler(conn); - conn->flags &= ~CONN_FLAG_IN_HANDLER; + connDecrRefs(conn); if (conn->flags & CONN_FLAG_CLOSE_SCHEDULED) { - connClose(conn); + if (!connHasRefs(conn)) connClose(conn); return 0; } return 1; From 9895f32dfb32ed391916783b9a8504cb04ff7cae Mon Sep 17 00:00:00 2001 From: Yossi Gottlieb Date: Sun, 22 Mar 2020 14:46:16 +0200 Subject: [PATCH 0189/1280] Cluster: fix misleading accept errors. --- src/cluster.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/cluster.c b/src/cluster.c index 72755823a..a2e9ff5b6 100644 --- a/src/cluster.c +++ b/src/cluster.c @@ -681,9 +681,10 @@ void clusterAcceptHandler(aeEventLoop *el, int fd, void *privdata, int mask) { * or schedule it for later depending on connection implementation. */ if (connAccept(conn, clusterConnAcceptHandler) == C_ERR) { - serverLog(LL_VERBOSE, - "Error accepting cluster node connection: %s", - connGetLastError(conn)); + if (connGetState(conn) == CONN_STATE_ERROR) + serverLog(LL_VERBOSE, + "Error accepting cluster node connection: %s", + connGetLastError(conn)); connClose(conn); return; } From 06391e27ce77199517ea5681a50fe9f423cd8cd7 Mon Sep 17 00:00:00 2001 From: Yossi Gottlieb Date: Sun, 22 Mar 2020 14:47:44 +0200 Subject: [PATCH 0190/1280] Fix crashes related to failed/rejected accepts. --- src/networking.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/networking.c b/src/networking.c index 69d59a59b..a550e4040 100644 --- a/src/networking.c +++ b/src/networking.c @@ -786,7 +786,7 @@ void clientAcceptHandler(connection *conn) { serverLog(LL_WARNING, "Error accepting a client connection: %s", connGetLastError(conn)); - freeClient(c); + freeClientAsync(c); return; } @@ -828,7 +828,7 @@ void clientAcceptHandler(connection *conn) { /* Nothing to do, Just to avoid the warning... */ } server.stat_rejected_conn++; - freeClient(c); + freeClientAsync(c); return; } } @@ -887,9 +887,10 @@ static void acceptCommonHandler(connection *conn, int flags, char *ip) { */ if (connAccept(conn, clientAcceptHandler) == C_ERR) { char conninfo[100]; - serverLog(LL_WARNING, - "Error accepting a client connection: %s (conn: %s)", - connGetLastError(conn), connGetInfo(conn, conninfo, sizeof(conninfo))); + if (connGetState(conn) == CONN_STATE_ERROR) + serverLog(LL_WARNING, + "Error accepting a client connection: %s (conn: %s)", + connGetLastError(conn), connGetInfo(conn, conninfo, sizeof(conninfo))); freeClient(connGetPrivateData(conn)); return; } From d286c1434af4fce6447784bfbe8711316c083e4f Mon Sep 17 00:00:00 2001 From: hwware Date: Mon, 23 Mar 2020 01:04:49 -0400 Subject: [PATCH 0191/1280] clean CLIENT_TRACKING_CACHING flag when disabled caching --- src/tracking.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tracking.c b/src/tracking.c index 45f83103a..5c1f48cba 100644 --- a/src/tracking.c +++ b/src/tracking.c @@ -94,7 +94,7 @@ void disableTracking(client *c) { server.tracking_clients--; c->flags &= ~(CLIENT_TRACKING|CLIENT_TRACKING_BROKEN_REDIR| CLIENT_TRACKING_BCAST|CLIENT_TRACKING_OPTIN| - CLIENT_TRACKING_OPTOUT); + CLIENT_TRACKING_OPTOUT|CLIENT_TRACKING_CACHING); } } From a2c5f8cfd55999479eb59652ba32f7e940af13ab Mon Sep 17 00:00:00 2001 From: hwware Date: Mon, 23 Mar 2020 01:07:46 -0400 Subject: [PATCH 0192/1280] remove redundant Semicolon --- src/tracking.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tracking.c b/src/tracking.c index 5c1f48cba..6f7929430 100644 --- a/src/tracking.c +++ b/src/tracking.c @@ -271,7 +271,7 @@ void trackingInvalidateKey(robj *keyobj) { trackingRememberKeyToBroadcast(sdskey,sdslen(sdskey)); rax *ids = raxFind(TrackingTable,(unsigned char*)sdskey,sdslen(sdskey)); - if (ids == raxNotFound) return;; + if (ids == raxNotFound) return; raxIterator ri; raxStart(&ri,ids); From 6a2a1d9a006743a2c46ac1323272efce14db337c Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 23 Mar 2020 11:17:50 +0100 Subject: [PATCH 0193/1280] Modules: updated function doc after #7003. --- src/module.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/module.c b/src/module.c index d6b055f78..6f61a5ca8 100644 --- a/src/module.c +++ b/src/module.c @@ -1795,7 +1795,12 @@ int RM_GetSelectedDb(RedisModuleCtx *ctx) { * current request context (whether the client is a Lua script or in a MULTI), * and about the Redis instance in general, i.e replication and persistence. * - * The available flags are: + * It is possible to call this function even with a NULL context, however + * in this case the following flags will not be reported: + * + * * LUA, MULTI, REPLICATED, DIRTY (see below for more info). + * + * Available flags and their meaning: * * * REDISMODULE_CTX_FLAGS_LUA: The command is running in a Lua script * From f3e021943f055a13409f55843a62b918b9dfda87 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 23 Mar 2020 11:28:09 +0100 Subject: [PATCH 0194/1280] Minor changes to BITFIELD_RO PR #6951. --- src/bitops.c | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/bitops.c b/src/bitops.c index ffb330013..d4e82c937 100644 --- a/src/bitops.c +++ b/src/bitops.c @@ -902,8 +902,8 @@ void bitposCommand(client *c) { * OVERFLOW [WRAP|SAT|FAIL] */ -#define BITFIELD_COMMON (1<<0) -#define BITFIELD_READONLY (1<<1) +#define BITFIELD_FLAG_NONE 0 +#define BITFIELD_FLAG_READONLY (1<<0) struct bitfieldOp { uint64_t offset; /* Bitfield offset. */ @@ -914,6 +914,9 @@ struct bitfieldOp { int sign; /* True if signed, otherwise unsigned op. */ }; +/* This implements both the BITFIELD command and the BITFIELD_RO command + * when flags is set to BITFIELD_FLAG_READONLY: in this case only the + * GET subcommand is allowed, other subcommands will return an error. */ void bitfieldGeneric(client *c, int flags) { robj *o; size_t bitoffset; @@ -1002,9 +1005,9 @@ void bitfieldGeneric(client *c, int flags) { return; } } else { - if (flags & BITFIELD_READONLY) { + if (flags & BITFIELD_FLAG_READONLY) { zfree(ops); - addReplyError(c, "bitfield_ro only support get subcommand"); + addReplyError(c, "BITFIELD_RO only support the GET subcommand"); return; } @@ -1140,9 +1143,9 @@ void bitfieldGeneric(client *c, int flags) { } void bitfieldCommand(client *c) { - bitfieldGeneric(c, BITFIELD_COMMON); + bitfieldGeneric(c, BITFIELD_FLAG_NONE); } void bitfieldroCommand(client *c) { - bitfieldGeneric(c, BITFIELD_READONLY); + bitfieldGeneric(c, BITFIELD_FLAG_READONLY); } From 0558a0b35e9b8e330dcbd6d87058a095ab0ad5a4 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 23 Mar 2020 11:47:37 +0100 Subject: [PATCH 0195/1280] Abort transactions after -READONLY error. Fix #7014. --- src/server.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/server.c b/src/server.c index f5fb339f9..ddc90b3dd 100644 --- a/src/server.c +++ b/src/server.c @@ -3509,6 +3509,7 @@ int processCommand(client *c) { !(c->flags & CLIENT_MASTER) && c->cmd->flags & CMD_WRITE) { + flagTransaction(c); addReply(c, shared.roslaveerr); return C_OK; } From fc92fa780b2526d82ea0ad89ff4cc3a2f0d76f84 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 23 Mar 2020 12:00:46 +0100 Subject: [PATCH 0196/1280] Fix BITFIELD_RO test. --- src/bitops.c | 2 +- tests/unit/bitfield.tcl | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/bitops.c b/src/bitops.c index d4e82c937..f78e4fd34 100644 --- a/src/bitops.c +++ b/src/bitops.c @@ -1007,7 +1007,7 @@ void bitfieldGeneric(client *c, int flags) { } else { if (flags & BITFIELD_FLAG_READONLY) { zfree(ops); - addReplyError(c, "BITFIELD_RO only support the GET subcommand"); + addReplyError(c, "BITFIELD_RO only supports the GET subcommand"); return; } diff --git a/tests/unit/bitfield.tcl b/tests/unit/bitfield.tcl index 819d8f36d..1f2f6395e 100644 --- a/tests/unit/bitfield.tcl +++ b/tests/unit/bitfield.tcl @@ -207,7 +207,7 @@ start_server {tags {"repl"}} { set master_port [srv -1 port] set slave [srv 0 client] - test {setup slave} { + test {BITFIELD: setup slave} { $slave slaveof $master_host $master_port wait_for_condition 50 100 { [s 0 master_link_status] eq {up} @@ -216,7 +216,7 @@ start_server {tags {"repl"}} { } } - test {write on master, read on slave} { + test {BITFIELD: write on master, read on slave} { $master del bits assert_equal 0 [$master bitfield bits set u8 0 255] assert_equal 255 [$master bitfield bits set u8 0 100] @@ -224,9 +224,9 @@ start_server {tags {"repl"}} { assert_equal 100 [$slave bitfield_ro bits get u8 0] } - test {bitfield_ro with write option} { + test {BITFIELD_RO fails when write option is used} { catch {$slave bitfield_ro bits set u8 0 100 get u8 0} err - assert_match {*ERR bitfield_ro only support get subcommand*} $err + assert_match {*ERR BITFIELD_RO only supports the GET subcommand*} $err } } } From 8dfbee0f898e79b64298ade4e78ea7075db0626d Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 23 Mar 2020 16:17:35 +0100 Subject: [PATCH 0197/1280] Improve comments of replicationCacheMasterUsingMyself(). --- src/replication.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/replication.c b/src/replication.c index 31e14d7fe..49c38f73f 100644 --- a/src/replication.c +++ b/src/replication.c @@ -2725,9 +2725,14 @@ void replicationCacheMaster(client *c) { * current offset if no data was lost during the failover. So we use our * current replication ID and offset in order to synthesize a cached master. */ void replicationCacheMasterUsingMyself(void) { + /* This will be used to populate the field server.master->reploff + * by replicationCreateMasterClient(). We'll later set the created + * master as server.cached_master, so the replica will use such + * offset for PSYNC. */ + server.master_initial_offset = server.master_repl_offset; + /* The master client we create can be set to any DBID, because * the new master will start its replication stream with SELECT. */ - server.master_initial_offset = server.master_repl_offset; replicationCreateMasterClient(NULL,-1); /* Use our own ID / offset. */ From fd1e1935a6beea1b74932c74d88f958934d0d47f Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Mon, 23 Mar 2020 20:13:52 +0200 Subject: [PATCH 0198/1280] MULTI/EXEC during LUA script timeout are messed up Redis refusing to run MULTI or EXEC during script timeout may cause partial transactions to run. 1) if the client sends MULTI+commands+EXEC in pipeline without waiting for response, but these arrive to the shards partially while there's a busy script, and partially after it eventually finishes: we'll end up running only part of the transaction (since multi was ignored, and exec would fail). 2) similar to the above if EXEC arrives during busy script, it'll be ignored and the client state remains in a transaction. the 3rd test which i added for a case where MULTI and EXEC are ok, and only the body arrives during busy script was already handled correctly since processCommand calls flagTransaction --- src/server.c | 1 + tests/unit/multi.tcl | 72 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 73 insertions(+) diff --git a/src/server.c b/src/server.c index ddc90b3dd..5a54a3abb 100644 --- a/src/server.c +++ b/src/server.c @@ -3553,6 +3553,7 @@ int processCommand(client *c) { c->cmd->proc != authCommand && c->cmd->proc != helloCommand && c->cmd->proc != replconfCommand && + c->cmd->proc != multiCommand && c->cmd->proc != execCommand && c->cmd->proc != discardCommand && !(c->cmd->proc == shutdownCommand && c->argc == 2 && tolower(((char*)c->argv[1]->ptr)[0]) == 'n') && diff --git a/tests/unit/multi.tcl b/tests/unit/multi.tcl index 9fcef71d6..55f18bec8 100644 --- a/tests/unit/multi.tcl +++ b/tests/unit/multi.tcl @@ -320,4 +320,76 @@ start_server {tags {"multi"}} { $rd close r ping } {PONG} + + test {MULTI and script timeout} { + # check that if MULTI arrives during timeout, it is either refused, or + # allowed to pass, and we don't end up executing half of the transaction + set rd1 [redis_deferring_client] + set rd2 [redis_deferring_client] + r config set lua-time-limit 10 + r set xx 1 + $rd1 eval {while true do end} 0 + after 200 + catch { $rd2 multi; $rd2 read } e + catch { $rd2 incr xx; $rd2 read } e + r script kill + after 200 ; # Give some time to Lua to call the hook again... + catch { $rd2 incr xx; $rd2 read } e + catch { $rd2 exec; $rd2 read } e + set xx [r get xx] + # make sure that either the whole transcation passed or none of it (we actually expect none) + assert { $xx == 1 || $xx == 3} + # check that the connection is no longer in multi state + $rd2 ping asdf + set pong [$rd2 read] + assert_equal $pong "asdf" + } + + test {EXEC and script timeout} { + # check that if EXEC arrives during timeout, we don't end up executing + # half of the transaction, and also that we exit the multi state + set rd1 [redis_deferring_client] + set rd2 [redis_deferring_client] + r config set lua-time-limit 10 + r set xx 1 + catch { $rd2 multi; $rd2 read } e + catch { $rd2 incr xx; $rd2 read } e + $rd1 eval {while true do end} 0 + after 200 + catch { $rd2 incr xx; $rd2 read } e + catch { $rd2 exec; $rd2 read } e + r script kill + after 200 ; # Give some time to Lua to call the hook again... + set xx [r get xx] + # make sure that either the whole transcation passed or none of it (we actually expect none) + assert { $xx == 1 || $xx == 3} + # check that the connection is no longer in multi state + $rd2 ping asdf + set pong [$rd2 read] + assert_equal $pong "asdf" + } + + test {MULTI-EXEC body and script timeout} { + # check that we don't run an imcomplete transaction due to some commands + # arriving during busy script + set rd1 [redis_deferring_client] + set rd2 [redis_deferring_client] + r config set lua-time-limit 10 + r set xx 1 + catch { $rd2 multi; $rd2 read } e + catch { $rd2 incr xx; $rd2 read } e + $rd1 eval {while true do end} 0 + after 200 + catch { $rd2 incr xx; $rd2 read } e + r script kill + after 200 ; # Give some time to Lua to call the hook again... + catch { $rd2 exec; $rd2 read } e + set xx [r get xx] + # make sure that either the whole transcation passed or none of it (we actually expect none) + assert { $xx == 1 || $xx == 3} + # check that the connection is no longer in multi state + $rd2 ping asdf + set pong [$rd2 read] + assert_equal $pong "asdf" + } } From 27a14b7a32935a13795a00e9390171e881064998 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 25 Mar 2020 12:46:59 +0100 Subject: [PATCH 0199/1280] Explain why we allow transactions in -BUSY state. Related to #7022. --- src/server.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/server.c b/src/server.c index 5a54a3abb..65a01db57 100644 --- a/src/server.c +++ b/src/server.c @@ -3548,12 +3548,19 @@ int processCommand(client *c) { return C_OK; } - /* Lua script too slow? Only allow a limited number of commands. */ + /* Lua script too slow? Only allow a limited number of commands. + * Note that we need to allow the transactions commands, otherwise clients + * sending a transaction with pipelining without error checking, may have + * the MULTI plus a few initial commands refused, then the timeout + * condition resolves, and the bottom-half of the transaction gets + * executed, see Github PR #7022. */ if (server.lua_timedout && c->cmd->proc != authCommand && c->cmd->proc != helloCommand && c->cmd->proc != replconfCommand && - c->cmd->proc != multiCommand && c->cmd->proc != execCommand && c->cmd->proc != discardCommand && + c->cmd->proc != multiCommand && + c->cmd->proc != execCommand && + c->cmd->proc != discardCommand && !(c->cmd->proc == shutdownCommand && c->argc == 2 && tolower(((char*)c->argv[1]->ptr)[0]) == 'n') && From 21976106a9e1ceefcf29e7965dd7ddb34916604f Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 24 Mar 2020 11:02:40 +0100 Subject: [PATCH 0200/1280] PSYNC2: meaningful offset implemented. A very commonly signaled operational problem with Redis master-replicas sets is that, once the master becomes unavailable for some reason, especially because of network problems, many times it wont be able to perform a partial resynchronization with the new master, once it rejoins the partition, for the following reason: 1. The master becomes isolated, however it keeps sending PINGs to the replicas. Such PINGs will never be received since the link connection is actually already severed. 2. On the other side, one of the replicas will turn into the new master, setting its secondary replication ID offset to the one of the last command received from the old master: this offset will not include the PINGs sent by the master once the link was already disconnected. 3. When the master rejoins the partion and is turned into a replica, its offset will be too advanced because of the PINGs, so a PSYNC will fail, and a full synchronization will be required. Related to issue #7002 and other discussion we had in the past around this problem. --- src/replication.c | 36 +++++++++++++++++++++++++++++++++++- src/server.c | 4 ++++ src/server.h | 1 + 3 files changed, 40 insertions(+), 1 deletion(-) diff --git a/src/replication.c b/src/replication.c index 49c38f73f..e62afb2fb 100644 --- a/src/replication.c +++ b/src/replication.c @@ -162,6 +162,7 @@ void feedReplicationBacklog(void *ptr, size_t len) { unsigned char *p = ptr; server.master_repl_offset += len; + server.master_repl_meaningful_offset = server.master_repl_offset; /* This is a circular buffer, so write as much data we can at every * iteration and rewind the "idx" index if we reach the limit. */ @@ -1768,6 +1769,7 @@ void readSyncBulkPayload(connection *conn) { * we are starting a new history. */ memcpy(server.replid,server.master->replid,sizeof(server.replid)); server.master_repl_offset = server.master->reploff; + server.master_repl_meaningful_offset = server.master->reploff; clearReplicationId2(); /* Let's create the replication backlog if needed. Slaves need to @@ -2725,12 +2727,37 @@ void replicationCacheMaster(client *c) { * current offset if no data was lost during the failover. So we use our * current replication ID and offset in order to synthesize a cached master. */ void replicationCacheMasterUsingMyself(void) { + serverLog(LL_NOTICE, + "Before turning into a replica, using my own master parameters " + "to synthesize a cached master: I may be able to synchronize with " + "the new master with just a partial transfer."); + /* This will be used to populate the field server.master->reploff * by replicationCreateMasterClient(). We'll later set the created * master as server.cached_master, so the replica will use such * offset for PSYNC. */ server.master_initial_offset = server.master_repl_offset; + /* However if the "meaningful" offset, that is the offset without + * the final PINGs in the stream, is different, use this instead: + * often when the master is no longer reachable, replicas will never + * receive the PINGs, however the master will end with an incremented + * offset because of the PINGs and will not be able to incrementally + * PSYNC with the new master. */ + if (server.master_repl_offset > server.master_repl_meaningful_offset) { + long long delta = server.master_repl_offset - + server.master_repl_meaningful_offset; + serverLog(LL_NOTICE, + "Using the meaningful offset %lld instead of %lld to exclude " + "the final PINGs (%lld bytes difference)", + server.master_repl_meaningful_offset, + server.master_repl_offset, + delta); + server.master_initial_offset = server.master_repl_meaningful_offset; + server.repl_backlog_histlen -= delta; + if (server.repl_backlog_histlen < 0) server.repl_backlog_histlen = 0; + } + /* The master client we create can be set to any DBID, because * the new master will start its replication stream with SELECT. */ replicationCreateMasterClient(NULL,-1); @@ -2742,7 +2769,6 @@ void replicationCacheMasterUsingMyself(void) { unlinkClient(server.master); server.cached_master = server.master; server.master = NULL; - serverLog(LL_NOTICE,"Before turning into a replica, using my master parameters to synthesize a cached master: I may be able to synchronize with the new master with just a partial transfer."); } /* Free a cached master, called when there are no longer the conditions for @@ -3122,10 +3148,18 @@ void replicationCron(void) { clientsArePaused(); if (!manual_failover_in_progress) { + long long before_ping = server.master_repl_meaningful_offset; ping_argv[0] = createStringObject("PING",4); replicationFeedSlaves(server.slaves, server.slaveseldb, ping_argv, 1); decrRefCount(ping_argv[0]); + /* The server.master_repl_meaningful_offset variable represents + * the offset of the replication stream without the pending PINGs. + * This is useful to set the right replication offset for PSYNC + * when the master is turned into a replica. Otherwise pending + * PINGs may not allow it to perform an incremental sync with the + * new master. */ + server.master_repl_meaningful_offset = before_ping; } } diff --git a/src/server.c b/src/server.c index 65a01db57..84439461e 100644 --- a/src/server.c +++ b/src/server.c @@ -2355,6 +2355,7 @@ void initServerConfig(void) { server.repl_syncio_timeout = CONFIG_REPL_SYNCIO_TIMEOUT; server.repl_down_since = 0; /* Never connected, repl is down since EVER. */ server.master_repl_offset = 0; + server.master_repl_meaningful_offset = 0; /* Replication partial resync backlog */ server.repl_backlog = NULL; @@ -4398,6 +4399,7 @@ sds genRedisInfoString(const char *section) { "master_replid:%s\r\n" "master_replid2:%s\r\n" "master_repl_offset:%lld\r\n" + "master_repl_meaningful_offset:%lld\r\n" "second_repl_offset:%lld\r\n" "repl_backlog_active:%d\r\n" "repl_backlog_size:%lld\r\n" @@ -4406,6 +4408,7 @@ sds genRedisInfoString(const char *section) { server.replid, server.replid2, server.master_repl_offset, + server.master_repl_meaningful_offset, server.second_replid_offset, server.repl_backlog != NULL, server.repl_backlog_size, @@ -4783,6 +4786,7 @@ void loadDataFromDisk(void) { { memcpy(server.replid,rsi.repl_id,sizeof(server.replid)); server.master_repl_offset = rsi.repl_offset; + server.master_repl_meaningful_offset = rsi.repl_offset; /* If we are a slave, create a cached master from this * information, in order to allow partial resynchronizations * with masters. */ diff --git a/src/server.h b/src/server.h index ed4707d66..818fbc3b3 100644 --- a/src/server.h +++ b/src/server.h @@ -1241,6 +1241,7 @@ struct redisServer { char replid[CONFIG_RUN_ID_SIZE+1]; /* My current replication ID. */ char replid2[CONFIG_RUN_ID_SIZE+1]; /* replid inherited from master*/ long long master_repl_offset; /* My current replication offset */ + long long master_repl_meaningful_offset; /* Offset minus latest PINGs. */ long long second_replid_offset; /* Accept offsets up to this for replid2. */ int slaveseldb; /* Last SELECTed DB in replication output */ int repl_ping_slave_period; /* Master pings the slave every N seconds */ From 3734ba96baebca9a56b6fb802457298f70e365c6 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 25 Mar 2020 15:43:34 +0100 Subject: [PATCH 0201/1280] PSYNC2: meaningful offset test. --- tests/integration/psync2-pingoff.tcl | 61 ++++++++++++++++++++++++++++ tests/test_helper.tcl | 1 + 2 files changed, 62 insertions(+) create mode 100644 tests/integration/psync2-pingoff.tcl diff --git a/tests/integration/psync2-pingoff.tcl b/tests/integration/psync2-pingoff.tcl new file mode 100644 index 000000000..1cea290e7 --- /dev/null +++ b/tests/integration/psync2-pingoff.tcl @@ -0,0 +1,61 @@ +# Test the meaningful offset implementation to make sure masters +# are able to PSYNC with replicas even if the replication stream +# has pending PINGs at the end. + +start_server {tags {"psync2"}} { +start_server {} { + # Config + set debug_msg 0 ; # Enable additional debug messages + + for {set j 0} {$j < 2} {incr j} { + set R($j) [srv [expr 0-$j] client] + set R_host($j) [srv [expr 0-$j] host] + set R_port($j) [srv [expr 0-$j] port] + $R($j) CONFIG SET repl-ping-replica-period 1 + if {$debug_msg} {puts "Log file: [srv [expr 0-$j] stdout]"} + } + + # Setup replication + test "PSYNC2 meaningful offset: setup" { + $R(1) replicaof $R_host(0) $R_port(0) + $R(0) set foo bar + wait_for_condition 50 1000 { + [$R(0) dbsize] == 1 && [$R(1) dbsize] == 1 + } else { + fail "Replicas not replicating from master" + } + } + + test "PSYNC2 meaningful offset: write and wait replication" { + $R(0) INCR counter + $R(0) INCR counter + $R(0) INCR counter + wait_for_condition 50 1000 { + [$R(0) GET counter] eq [$R(1) GET counter] + } else { + fail "Master and replica don't agree about counter" + } + } + + # In this test we'll make sure the replica will get stuck, but with + # an active connection: this way the master will continue to send PINGs + # every second (we modified the PING period earlier) + test "PSYNC2 meaningful offset: pause replica and promote it" { + $R(1) MULTI + $R(1) DEBUG SLEEP 5 + $R(1) SLAVEOF NO ONE + $R(1) EXEC + $R(1) ping ; # Wait for it to return back available + } + + test "Make the old master a replica of the new one and check conditions" { + set sync_partial [status $R(1) sync_partial_ok] + assert {$sync_partial == 0} + $R(0) REPLICAOF $R_host(1) $R_port(1) + wait_for_condition 50 1000 { + [status $R(1) sync_partial_ok] == 1 + } else { + fail "The new master was not able to partial sync" + } + } +}} diff --git a/tests/test_helper.tcl b/tests/test_helper.tcl index 4dbead193..5cb43104b 100644 --- a/tests/test_helper.tcl +++ b/tests/test_helper.tcl @@ -47,6 +47,7 @@ set ::all_tests { integration/logging integration/psync2 integration/psync2-reg + integration/psync2-pingoff unit/pubsub unit/slowlog unit/scripting From 1ed18d7cd765a8429abc4d16dc66c2bdc2502b26 Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Wed, 4 Dec 2019 17:29:29 +0200 Subject: [PATCH 0202/1280] AOFRW on an empty stream created with MKSTREAM loads badkly the AOF will be loaded successfully, but the stream will be missing, i.e inconsistencies with the original db. this was because XADD with id of 0-0 would error. add a test to reproduce. --- src/aof.c | 3 ++- tests/unit/type/stream-cgroups.tcl | 13 +++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/src/aof.c b/src/aof.c index 8ab9349f0..6bb239252 100644 --- a/src/aof.c +++ b/src/aof.c @@ -1212,12 +1212,13 @@ int rewriteStreamObject(rio *r, robj *key, robj *o) { /* Use the XADD MAXLEN 0 trick to generate an empty stream if * the key we are serializing is an empty string, which is possible * for the Stream type. */ + id.ms = 0; id.seq = 1; if (rioWriteBulkCount(r,'*',7) == 0) return 0; if (rioWriteBulkString(r,"XADD",4) == 0) return 0; if (rioWriteBulkObject(r,key) == 0) return 0; if (rioWriteBulkString(r,"MAXLEN",6) == 0) return 0; if (rioWriteBulkString(r,"0",1) == 0) return 0; - if (rioWriteBulkStreamID(r,&s->last_id) == 0) return 0; + if (rioWriteBulkStreamID(r,&id) == 0) return 0; if (rioWriteBulkString(r,"x",1) == 0) return 0; if (rioWriteBulkString(r,"y",1) == 0) return 0; } diff --git a/tests/unit/type/stream-cgroups.tcl b/tests/unit/type/stream-cgroups.tcl index 072ed14d6..6b9a4a9cd 100644 --- a/tests/unit/type/stream-cgroups.tcl +++ b/tests/unit/type/stream-cgroups.tcl @@ -311,4 +311,17 @@ start_server { } } } + + start_server {tags {"stream"} overrides {appendonly yes aof-use-rdb-preamble no}} { + test {Empty stream with no lastid can be rewrite into AOF correctly} { + r XGROUP CREATE mystream group-name $ MKSTREAM + assert {[dict get [r xinfo stream mystream] length] == 0} + set grpinfo [r xinfo groups mystream] + r bgrewriteaof + waitForBgrewriteaof r + r debug loadaof + assert {[dict get [r xinfo stream mystream] length] == 0} + assert {[r xinfo groups mystream] == $grpinfo} + } + } } From 81c1e22b8d23705967670d392b5625b77cb8f03b Mon Sep 17 00:00:00 2001 From: Valentino Geron Date: Wed, 25 Mar 2020 21:54:14 +0200 Subject: [PATCH 0203/1280] XACK should be executed in a "all or nothing" fashion. First, we must parse the IDs, so that we abort ASAP. The return value of this command cannot be an error if the client successfully acknowledged some messages, so it should be executed in a "all or nothing" fashion. --- src/t_stream.c | 12 +++++++++++- tests/unit/type/stream-cgroups.tcl | 12 ++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/src/t_stream.c b/src/t_stream.c index 557d1d642..2bf6a26a7 100644 --- a/src/t_stream.c +++ b/src/t_stream.c @@ -1921,11 +1921,21 @@ void xackCommand(client *c) { return; } + /* Start parsing the IDs, so that we abort ASAP if there is a syntax + * error: the return value of this command cannot be an error in case + * the client successfully acknowledged some messages, so it should be + * executed in a "all or nothing" fashion. */ + for (int j = 3; j < c->argc; j++) { + streamID id; + if (streamParseStrictIDOrReply(c,c->argv[j],&id,0) != C_OK) return; + } + int acknowledged = 0; for (int j = 3; j < c->argc; j++) { streamID id; unsigned char buf[sizeof(streamID)]; - if (streamParseStrictIDOrReply(c,c->argv[j],&id,0) != C_OK) return; + if (streamParseStrictIDOrReply(c,c->argv[j],&id,0) != C_OK) + serverPanic("StreamID invalid after check. Should not be possible."); streamEncodeID(buf,&id); /* Lookup the ID in the group PEL: it will have a reference to the diff --git a/tests/unit/type/stream-cgroups.tcl b/tests/unit/type/stream-cgroups.tcl index 072ed14d6..2fc0c51cd 100644 --- a/tests/unit/type/stream-cgroups.tcl +++ b/tests/unit/type/stream-cgroups.tcl @@ -93,6 +93,18 @@ start_server { assert {[r XACK mystream mygroup $id1 $id2] eq 1} } + test {XACK should fail if got at least one invalid ID} { + r del mystream + r xgroup create s g $ MKSTREAM + r xadd s * f1 v1 + set c [llength [lindex [r xreadgroup group g c streams s >] 0 1]] + assert {$c == 1} + set pending [r xpending s g - + 10 c] + set id1 [lindex $pending 0 0] + assert_error "*Invalid stream ID specified*" {r xack s g $id1 invalid-id} + assert {[r xack s g $id1] eq 1} + } + test {PEL NACK reassignment after XGROUP SETID event} { r del events r xadd events * f1 v1 From 53a7041535aadbb5104f67932965476189207906 Mon Sep 17 00:00:00 2001 From: Valentino Geron Date: Thu, 26 Mar 2020 11:49:21 +0200 Subject: [PATCH 0204/1280] XREAD and XREADGROUP should not be allowed from scripts when BLOCK option is being used --- src/server.c | 4 ++-- src/t_stream.c | 5 +++++ tests/unit/scripting.tcl | 11 +++++++++++ 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/src/server.c b/src/server.c index 84439461e..4fc7e0103 100644 --- a/src/server.c +++ b/src/server.c @@ -947,11 +947,11 @@ struct redisCommand redisCommandTable[] = { 0,NULL,1,1,1,0,0,0}, {"xread",xreadCommand,-4, - "read-only no-script @stream @blocking", + "read-only @stream @blocking", 0,xreadGetKeys,1,1,1,0,0,0}, {"xreadgroup",xreadCommand,-7, - "write no-script @stream @blocking", + "write @stream @blocking", 0,xreadGetKeys,1,1,1,0,0,0}, {"xgroup",xgroupCommand,-2, diff --git a/src/t_stream.c b/src/t_stream.c index 557d1d642..157f1eaeb 100644 --- a/src/t_stream.c +++ b/src/t_stream.c @@ -1373,6 +1373,11 @@ void xreadCommand(client *c) { int moreargs = c->argc-i-1; char *o = c->argv[i]->ptr; if (!strcasecmp(o,"BLOCK") && moreargs) { + if (c->flags & CLIENT_LUA) { + /* There is no sense to use BLOCK option within LUA */ + addReplyErrorFormat(c, "%s command is not allowed with BLOCK option from scripts", (char *)c->argv[0]->ptr); + return; + } i++; if (getTimeoutFromObjectOrReply(c,c->argv[i],&timeout, UNIT_MILLISECONDS) != C_OK) return; diff --git a/tests/unit/scripting.tcl b/tests/unit/scripting.tcl index fb36d0b80..8b364b287 100644 --- a/tests/unit/scripting.tcl +++ b/tests/unit/scripting.tcl @@ -146,6 +146,17 @@ start_server {tags {"scripting"}} { set e } {*not allowed*} + test {EVAL - Scripts can't run XREAD and XREADGROUP with BLOCK option} { + r del s + r xgroup create s g $ MKSTREAM + set res [r eval {return redis.pcall('xread','STREAMS','s','$')} 1 s] + assert {$res eq {}} + assert_error "*xread command is not allowed with BLOCK option from scripts" {r eval {return redis.pcall('xread','BLOCK',0,'STREAMS','s','$')} 1 s} + set res [r eval {return redis.pcall('xreadgroup','group','g','c','STREAMS','s','>')} 1 s] + assert {$res eq {}} + assert_error "*xreadgroup command is not allowed with BLOCK option from scripts" {r eval {return redis.pcall('xreadgroup','group','g','c','BLOCK',0,'STREAMS','s','>')} 1 s} + } + test {EVAL - Scripts can't run certain commands} { set e {} r debug lua-always-replicate-commands 0 From 94901f61b85e754a8ba4e1b8eee2e1ee4b04bca1 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 27 Mar 2020 16:19:56 +0100 Subject: [PATCH 0205/1280] PSYNC2: fix backlog_idx when adjusting for meaningful offset See #7002. --- src/replication.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/replication.c b/src/replication.c index e62afb2fb..20f9a2129 100644 --- a/src/replication.c +++ b/src/replication.c @@ -2755,6 +2755,9 @@ void replicationCacheMasterUsingMyself(void) { delta); server.master_initial_offset = server.master_repl_meaningful_offset; server.repl_backlog_histlen -= delta; + server.repl_backlog_idx = + (server.repl_backlog_idx + (server.repl_backlog_size - delta)) % + server.repl_backlog_size; if (server.repl_backlog_histlen < 0) server.repl_backlog_histlen = 0; } From 86bd36d93aa62c19c3f94f3d68e60ab4f4153c85 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 26 Mar 2020 11:33:18 +0100 Subject: [PATCH 0206/1280] Precise timeouts: refactor unblocking on timeout. --- src/server.c | 41 +++++++++++++++++++++++++++++------------ src/server.h | 3 +++ 2 files changed, 32 insertions(+), 12 deletions(-) diff --git a/src/server.c b/src/server.c index 84439461e..db56f6a0d 100644 --- a/src/server.c +++ b/src/server.c @@ -1502,6 +1502,23 @@ long long getInstantaneousMetric(int metric) { return sum / STATS_METRIC_SAMPLES; } +/* Check if this blocked client timedout (does nothing if the client is + * not blocked right now). If so send a reply, unblock it, and return 1. + * Otherwise 0 is returned and no operation is performed. */ +int checkBlockedClientTimeout(client *c, mstime_t now) { + if (c->flags & CLIENT_BLOCKED && + c->bpop.timeout != 0 + && c->bpop.timeout < now) + { + /* Handle blocking operation specific timeout. */ + replyToBlockedClientTimedOut(c); + unblockClient(c); + return 1; + } else { + return 0; + } +} + /* Check for timeouts. Returns non-zero if the client was terminated. * The function gets the current time in milliseconds as argument since * it gets called multiple times in a loop, so calling gettimeofday() for @@ -1510,10 +1527,11 @@ int clientsCronHandleTimeout(client *c, mstime_t now_ms) { time_t now = now_ms/1000; if (server.maxidletime && - !(c->flags & CLIENT_SLAVE) && /* no timeout for slaves and monitors */ - !(c->flags & CLIENT_MASTER) && /* no timeout for masters */ - !(c->flags & CLIENT_BLOCKED) && /* no timeout for BLPOP */ - !(c->flags & CLIENT_PUBSUB) && /* no timeout for Pub/Sub clients */ + /* This handles the idle clients connection timeout if set. */ + !(c->flags & CLIENT_SLAVE) && /* No timeout for slaves and monitors */ + !(c->flags & CLIENT_MASTER) && /* No timeout for masters */ + !(c->flags & CLIENT_BLOCKED) && /* No timeout for BLPOP */ + !(c->flags & CLIENT_PUBSUB) && /* No timeout for Pub/Sub clients */ (now - c->lastinteraction > server.maxidletime)) { serverLog(LL_VERBOSE,"Closing idle client"); @@ -1522,15 +1540,14 @@ int clientsCronHandleTimeout(client *c, mstime_t now_ms) { } else if (c->flags & CLIENT_BLOCKED) { /* Blocked OPS timeout is handled with milliseconds resolution. * However note that the actual resolution is limited by - * server.hz. */ + * server.hz. So for short timeouts (less than SERVER_SHORT_TIMEOUT + * milliseconds) we populate a Radix tree and handle such timeouts + * in clientsHandleShortTimeout(). */ + if (checkBlockedClientTimeout(c,now_ms)) return 0; - if (c->bpop.timeout != 0 && c->bpop.timeout < now_ms) { - /* Handle blocking operation specific timeout. */ - replyToBlockedClientTimedOut(c); - unblockClient(c); - } else if (server.cluster_enabled) { - /* Cluster: handle unblock & redirect of clients blocked - * into keys no longer served by this server. */ + /* Cluster: handle unblock & redirect of clients blocked + * into keys no longer served by this server. */ + if (server.cluster_enabled) { if (clusterRedirectBlockedClientIfNeeded(c)) unblockClient(c); } diff --git a/src/server.h b/src/server.h index 818fbc3b3..93552b36e 100644 --- a/src/server.h +++ b/src/server.h @@ -277,6 +277,9 @@ typedef long long ustime_t; /* microsecond time type. */ buffer configuration. Just the first three: normal, slave, pubsub. */ +/* Other client related defines. */ +#define CLIENT_SHORT_TIMEOUT 2000 /* See clientsHandleShortTimeout(). */ + /* Slave replication state. Used in server.repl_state for slaves to remember * what to do next. */ #define REPL_STATE_NONE 0 /* No active replication */ From f232cb66705b3d08dd153385c8ae746156d791a1 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 26 Mar 2020 13:28:39 +0100 Subject: [PATCH 0207/1280] Precise timeouts: working initial implementation. --- src/blocked.c | 1 + src/server.c | 135 +++++++++++++++++++++++++++++++++++++++----------- src/server.h | 2 + 3 files changed, 110 insertions(+), 28 deletions(-) diff --git a/src/blocked.c b/src/blocked.c index 06aa5850e..c470cba00 100644 --- a/src/blocked.c +++ b/src/blocked.c @@ -619,6 +619,7 @@ void blockForKeys(client *c, int btype, robj **keys, int numkeys, mstime_t timeo listAddNodeTail(l,c); } blockClient(c,btype); + addClientToShortTimeoutTable(c); } /* Unblock a client that's waiting in a blocking operation such as BLPOP. diff --git a/src/server.c b/src/server.c index db56f6a0d..bd63c4b4e 100644 --- a/src/server.c +++ b/src/server.c @@ -1473,34 +1473,7 @@ int allPersistenceDisabled(void) { return server.saveparamslen == 0 && server.aof_state == AOF_OFF; } -/* ======================= Cron: called every 100 ms ======================== */ - -/* Add a sample to the operations per second array of samples. */ -void trackInstantaneousMetric(int metric, long long current_reading) { - long long t = mstime() - server.inst_metric[metric].last_sample_time; - long long ops = current_reading - - server.inst_metric[metric].last_sample_count; - long long ops_sec; - - ops_sec = t > 0 ? (ops*1000/t) : 0; - - server.inst_metric[metric].samples[server.inst_metric[metric].idx] = - ops_sec; - server.inst_metric[metric].idx++; - server.inst_metric[metric].idx %= STATS_METRIC_SAMPLES; - server.inst_metric[metric].last_sample_time = mstime(); - server.inst_metric[metric].last_sample_count = current_reading; -} - -/* Return the mean of all the samples. */ -long long getInstantaneousMetric(int metric) { - int j; - long long sum = 0; - - for (j = 0; j < STATS_METRIC_SAMPLES; j++) - sum += server.inst_metric[metric].samples[j]; - return sum / STATS_METRIC_SAMPLES; -} +/* ========================== Clients timeouts ============================= */ /* Check if this blocked client timedout (does nothing if the client is * not blocked right now). If so send a reply, unblock it, and return 1. @@ -1555,6 +1528,107 @@ int clientsCronHandleTimeout(client *c, mstime_t now_ms) { return 0; } +/* For shor timeouts, less than < CLIENT_SHORT_TIMEOUT milliseconds, we + * populate a radix tree of 128 bit keys composed as such: + * + * [8 byte big endian expire time]+[8 byte client ID] + * + * We don't do any cleanup in the Radix tree: when we run the clients that + * reached the timeout already, if they are no longer existing or no longer + * blocked with such timeout, we just go forward. + * + * Every time a client blocks with a short timeout, we add the client in + * the tree. In beforeSleep() we call clientsHandleShortTimeout() to run + * the tree and unblock the clients. + * + * Design hint: why we block only clients with short timeouts? For frugality: + * Clients blocking for 30 seconds usually don't need to be unblocked + * precisely, and anyway for the nature of Redis to *guarantee* unblock time + * precision is hard, so we can avoid putting a large number of clients in + * the radix tree without a good reason. This idea also has a role in memory + * usage as well given that we don't do cleanup, the shorter a client timeout, + * the less time it will stay in the radix tree. */ + +#define CLIENT_ST_KEYLEN 16 /* 8 bytes mstime + 8 bytes client ID. */ + +/* Given client ID and timeout, write the resulting radix tree key in buf. */ +void encodeTimeoutKey(unsigned char *buf, uint64_t timeout, uint64_t id) { + timeout = htonu64(timeout); + memcpy(buf,&timeout,sizeof(timeout)); + memcpy(buf+8,&id,sizeof(id)); +} + +/* Given a key encoded with encodeTimeoutKey(), resolve the fields and write + * the timeout into *toptr and the client ID into *idptr. */ +void decodeTimeoutKey(unsigned char *buf, uint64_t *toptr, uint64_t *idptr) { + memcpy(toptr,buf,sizeof(*toptr)); + *toptr = ntohu64(*toptr); + memcpy(idptr,buf+8,sizeof(*idptr)); +} + +/* Add the specified client id / timeout as a key in the radix tree we use + * to handle short timeouts. The client is not added to the list if its + * timeout is longer than CLIENT_SHORT_TIMEOUT milliseconds. */ +void addClientToShortTimeoutTable(client *c) { + if (c->bpop.timeout == 0 || + c->bpop.timeout - mstime() > CLIENT_SHORT_TIMEOUT) + { + return; + } + uint64_t timeout = c->bpop.timeout; + uint64_t id = c->id; + unsigned char buf[CLIENT_ST_KEYLEN]; + encodeTimeoutKey(buf,timeout,id); + raxTryInsert(server.clients_timeout_table,buf,sizeof(buf),NULL,NULL); +} + +/* This function is called in beforeSleep() in order to unblock ASAP clients + * that are waiting in blocking operations with a short timeout set. */ +void clientsHandleShortTimeout(void) { + uint64_t now = mstime(); + raxIterator ri; + raxStart(&ri,server.clients_timeout_table); + + while(raxNext(&ri)) { + uint64_t id, timeout; + decodeTimeoutKey(ri.key,&timeout,&id); + if (timeout >= now) break; /* All the timeouts are in the future. */ + client *c = lookupClientByID(id); + if (c) checkBlockedClientTimeout(c,now); + raxRemove(server.clients_timeout_table,ri.key,ri.key_len,NULL); + raxSeek(&ri,"^",NULL,0); + } +} + +/* ======================= Cron: called every 100 ms ======================== */ + +/* Add a sample to the operations per second array of samples. */ +void trackInstantaneousMetric(int metric, long long current_reading) { + long long t = mstime() - server.inst_metric[metric].last_sample_time; + long long ops = current_reading - + server.inst_metric[metric].last_sample_count; + long long ops_sec; + + ops_sec = t > 0 ? (ops*1000/t) : 0; + + server.inst_metric[metric].samples[server.inst_metric[metric].idx] = + ops_sec; + server.inst_metric[metric].idx++; + server.inst_metric[metric].idx %= STATS_METRIC_SAMPLES; + server.inst_metric[metric].last_sample_time = mstime(); + server.inst_metric[metric].last_sample_count = current_reading; +} + +/* Return the mean of all the samples. */ +long long getInstantaneousMetric(int metric) { + int j; + long long sum = 0; + + for (j = 0; j < STATS_METRIC_SAMPLES; j++) + sum += server.inst_metric[metric].samples[j]; + return sum / STATS_METRIC_SAMPLES; +} + /* The client query buffer is an sds.c string that can end with a lot of * free space not used, this function reclaims space if needed. * @@ -2109,11 +2183,15 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) { void beforeSleep(struct aeEventLoop *eventLoop) { UNUSED(eventLoop); + /* Handle precise timeouts of blocked clients. */ + clientsHandleShortTimeout(); + /* We should handle pending reads clients ASAP after event loop. */ handleClientsWithPendingReadsUsingThreads(); /* Handle TLS pending data. (must be done before flushAppendOnlyFile) */ tlsProcessPendingData(); + /* If tls still has pending unread data don't sleep at all. */ aeSetDontWait(server.el, tlsHasPendingData()); @@ -2738,6 +2816,7 @@ void initServer(void) { server.monitors = listCreate(); server.clients_pending_write = listCreate(); server.clients_pending_read = listCreate(); + server.clients_timeout_table = raxNew(); server.slaveseldb = -1; /* Force to emit the first SELECT command. */ server.unblocked_clients = listCreate(); server.ready_keys = listCreate(); diff --git a/src/server.h b/src/server.h index 93552b36e..6f417b730 100644 --- a/src/server.h +++ b/src/server.h @@ -1070,6 +1070,7 @@ struct redisServer { list *clients_pending_read; /* Client has pending read socket buffers. */ list *slaves, *monitors; /* List of slaves and MONITORs */ client *current_client; /* Current client executing the command. */ + rax *clients_timeout_table; /* Radix tree for clients with short timeout. */ long fixed_time_expire; /* If > 0, expire keys against server.mstime. */ rax *clients_index; /* Active clients dictionary by client ID. */ int clients_paused; /* True if clients are currently paused */ @@ -2139,6 +2140,7 @@ void disconnectAllBlockedClients(void); void handleClientsBlockedOnKeys(void); void signalKeyAsReady(redisDb *db, robj *key); void blockForKeys(client *c, int btype, robj **keys, int numkeys, mstime_t timeout, robj *target, streamID *ids); +void addClientToShortTimeoutTable(client *c); /* expire.c -- Handling of expired keys */ void activeExpireCycle(int type); From 36cba9bb8ed1a634e997e9dd586fba3fe9e02e9a Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 26 Mar 2020 14:37:00 +0100 Subject: [PATCH 0208/1280] Precise timeouts: fix bugs in initial implementation. --- src/blocked.c | 2 +- src/server.c | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/blocked.c b/src/blocked.c index c470cba00..84a5287d4 100644 --- a/src/blocked.c +++ b/src/blocked.c @@ -111,6 +111,7 @@ void blockClient(client *c, int btype) { c->btype = btype; server.blocked_clients++; server.blocked_clients_by_type[btype]++; + addClientToShortTimeoutTable(c); } /* This function is called in the beforeSleep() function of the event loop @@ -619,7 +620,6 @@ void blockForKeys(client *c, int btype, robj **keys, int numkeys, mstime_t timeo listAddNodeTail(l,c); } blockClient(c,btype); - addClientToShortTimeoutTable(c); } /* Unblock a client that's waiting in a blocking operation such as BLPOP. diff --git a/src/server.c b/src/server.c index bd63c4b4e..25af4e161 100644 --- a/src/server.c +++ b/src/server.c @@ -1588,6 +1588,7 @@ void clientsHandleShortTimeout(void) { uint64_t now = mstime(); raxIterator ri; raxStart(&ri,server.clients_timeout_table); + raxSeek(&ri,"^",NULL,0); while(raxNext(&ri)) { uint64_t id, timeout; @@ -1745,6 +1746,9 @@ void getExpansiveClientsInfo(size_t *in_usage, size_t *out_usage) { */ #define CLIENTS_CRON_MIN_ITERATIONS 5 void clientsCron(void) { + /* Unblock short timeout clients ASAP. */ + clientsHandleShortTimeout(); + /* Try to process at least numclients/server.hz of clients * per call. Since normally (if there are no big latency events) this * function is called server.hz times per second, in the average case we From c54576c63182eaa50714ebd8b322b9b9d10f11a2 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 26 Mar 2020 15:52:16 +0100 Subject: [PATCH 0209/1280] Precise timeouts: fast exit for clientsHandleShortTimeout(). --- src/server.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/server.c b/src/server.c index 25af4e161..7240794f1 100644 --- a/src/server.c +++ b/src/server.c @@ -1585,6 +1585,7 @@ void addClientToShortTimeoutTable(client *c) { /* This function is called in beforeSleep() in order to unblock ASAP clients * that are waiting in blocking operations with a short timeout set. */ void clientsHandleShortTimeout(void) { + if (raxSize(server.clients_timeout_table) == 0) return; uint64_t now = mstime(); raxIterator ri; raxStart(&ri,server.clients_timeout_table); From 0264a78063f6d69a4939bfb7a02655eb9c8186de Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 26 Mar 2020 16:02:26 +0100 Subject: [PATCH 0210/1280] Precise timeouts: use only radix tree for timeouts. --- src/blocked.c | 2 +- src/server.c | 46 +++++++++++++--------------------------------- src/server.h | 5 +---- 3 files changed, 15 insertions(+), 38 deletions(-) diff --git a/src/blocked.c b/src/blocked.c index 84a5287d4..443daec7f 100644 --- a/src/blocked.c +++ b/src/blocked.c @@ -111,7 +111,7 @@ void blockClient(client *c, int btype) { c->btype = btype; server.blocked_clients++; server.blocked_clients_by_type[btype]++; - addClientToShortTimeoutTable(c); + addClientToTimeoutTable(c); } /* This function is called in the beforeSleep() function of the event loop diff --git a/src/server.c b/src/server.c index 7240794f1..fc21a128e 100644 --- a/src/server.c +++ b/src/server.c @@ -1511,13 +1511,6 @@ int clientsCronHandleTimeout(client *c, mstime_t now_ms) { freeClient(c); return 1; } else if (c->flags & CLIENT_BLOCKED) { - /* Blocked OPS timeout is handled with milliseconds resolution. - * However note that the actual resolution is limited by - * server.hz. So for short timeouts (less than SERVER_SHORT_TIMEOUT - * milliseconds) we populate a Radix tree and handle such timeouts - * in clientsHandleShortTimeout(). */ - if (checkBlockedClientTimeout(c,now_ms)) return 0; - /* Cluster: handle unblock & redirect of clients blocked * into keys no longer served by this server. */ if (server.cluster_enabled) { @@ -1528,8 +1521,8 @@ int clientsCronHandleTimeout(client *c, mstime_t now_ms) { return 0; } -/* For shor timeouts, less than < CLIENT_SHORT_TIMEOUT milliseconds, we - * populate a radix tree of 128 bit keys composed as such: +/* For blocked clients timeouts we populate a radix tree of 128 bit keys + * composed as such: * * [8 byte big endian expire time]+[8 byte client ID] * @@ -1538,16 +1531,8 @@ int clientsCronHandleTimeout(client *c, mstime_t now_ms) { * blocked with such timeout, we just go forward. * * Every time a client blocks with a short timeout, we add the client in - * the tree. In beforeSleep() we call clientsHandleShortTimeout() to run - * the tree and unblock the clients. - * - * Design hint: why we block only clients with short timeouts? For frugality: - * Clients blocking for 30 seconds usually don't need to be unblocked - * precisely, and anyway for the nature of Redis to *guarantee* unblock time - * precision is hard, so we can avoid putting a large number of clients in - * the radix tree without a good reason. This idea also has a role in memory - * usage as well given that we don't do cleanup, the shorter a client timeout, - * the less time it will stay in the radix tree. */ + * the tree. In beforeSleep() we call clientsHandleTimeout() to run + * the tree and unblock the clients. */ #define CLIENT_ST_KEYLEN 16 /* 8 bytes mstime + 8 bytes client ID. */ @@ -1568,13 +1553,9 @@ void decodeTimeoutKey(unsigned char *buf, uint64_t *toptr, uint64_t *idptr) { /* Add the specified client id / timeout as a key in the radix tree we use * to handle short timeouts. The client is not added to the list if its - * timeout is longer than CLIENT_SHORT_TIMEOUT milliseconds. */ -void addClientToShortTimeoutTable(client *c) { - if (c->bpop.timeout == 0 || - c->bpop.timeout - mstime() > CLIENT_SHORT_TIMEOUT) - { - return; - } + * timeout is zero (block forever). */ +void addClientToTimeoutTable(client *c) { + if (c->bpop.timeout == 0) return; uint64_t timeout = c->bpop.timeout; uint64_t id = c->id; unsigned char buf[CLIENT_ST_KEYLEN]; @@ -1584,7 +1565,7 @@ void addClientToShortTimeoutTable(client *c) { /* This function is called in beforeSleep() in order to unblock ASAP clients * that are waiting in blocking operations with a short timeout set. */ -void clientsHandleShortTimeout(void) { +void clientsHandleTimeout(void) { if (raxSize(server.clients_timeout_table) == 0) return; uint64_t now = mstime(); raxIterator ri; @@ -1747,9 +1728,6 @@ void getExpansiveClientsInfo(size_t *in_usage, size_t *out_usage) { */ #define CLIENTS_CRON_MIN_ITERATIONS 5 void clientsCron(void) { - /* Unblock short timeout clients ASAP. */ - clientsHandleShortTimeout(); - /* Try to process at least numclients/server.hz of clients * per call. Since normally (if there are no big latency events) this * function is called server.hz times per second, in the average case we @@ -2189,7 +2167,7 @@ void beforeSleep(struct aeEventLoop *eventLoop) { UNUSED(eventLoop); /* Handle precise timeouts of blocked clients. */ - clientsHandleShortTimeout(); + clientsHandleTimeout(); /* We should handle pending reads clients ASAP after event loop. */ handleClientsWithPendingReadsUsingThreads(); @@ -4101,11 +4079,13 @@ sds genRedisInfoString(const char *section) { "client_recent_max_input_buffer:%zu\r\n" "client_recent_max_output_buffer:%zu\r\n" "blocked_clients:%d\r\n" - "tracking_clients:%d\r\n", + "tracking_clients:%d\r\n" + "clients_in_timeout_table:%lld\r\n", listLength(server.clients)-listLength(server.slaves), maxin, maxout, server.blocked_clients, - server.tracking_clients); + server.tracking_clients, + raxSize(server.clients_timeout_table)); } /* Memory */ diff --git a/src/server.h b/src/server.h index 6f417b730..7ccab28ba 100644 --- a/src/server.h +++ b/src/server.h @@ -277,9 +277,6 @@ typedef long long ustime_t; /* microsecond time type. */ buffer configuration. Just the first three: normal, slave, pubsub. */ -/* Other client related defines. */ -#define CLIENT_SHORT_TIMEOUT 2000 /* See clientsHandleShortTimeout(). */ - /* Slave replication state. Used in server.repl_state for slaves to remember * what to do next. */ #define REPL_STATE_NONE 0 /* No active replication */ @@ -2140,7 +2137,7 @@ void disconnectAllBlockedClients(void); void handleClientsBlockedOnKeys(void); void signalKeyAsReady(redisDb *db, robj *key); void blockForKeys(client *c, int btype, robj **keys, int numkeys, mstime_t timeout, robj *target, streamID *ids); -void addClientToShortTimeoutTable(client *c); +void addClientToTimeoutTable(client *c); /* expire.c -- Handling of expired keys */ void activeExpireCycle(int type); From 2ee65aff6b5d0d443cd703116e658e8d4d997c3b Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 26 Mar 2020 16:05:20 +0100 Subject: [PATCH 0211/1280] Precise timeouts: fix comments after functional change. --- src/server.c | 10 +++++----- src/server.h | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/server.c b/src/server.c index fc21a128e..2624cbecd 100644 --- a/src/server.c +++ b/src/server.c @@ -1530,7 +1530,7 @@ int clientsCronHandleTimeout(client *c, mstime_t now_ms) { * reached the timeout already, if they are no longer existing or no longer * blocked with such timeout, we just go forward. * - * Every time a client blocks with a short timeout, we add the client in + * Every time a client blocks with a timeout, we add the client in * the tree. In beforeSleep() we call clientsHandleTimeout() to run * the tree and unblock the clients. */ @@ -1552,8 +1552,8 @@ void decodeTimeoutKey(unsigned char *buf, uint64_t *toptr, uint64_t *idptr) { } /* Add the specified client id / timeout as a key in the radix tree we use - * to handle short timeouts. The client is not added to the list if its - * timeout is zero (block forever). */ + * to handle blocked clients timeouts. The client is not added to the list + * if its timeout is zero (block forever). */ void addClientToTimeoutTable(client *c) { if (c->bpop.timeout == 0) return; uint64_t timeout = c->bpop.timeout; @@ -1563,8 +1563,8 @@ void addClientToTimeoutTable(client *c) { raxTryInsert(server.clients_timeout_table,buf,sizeof(buf),NULL,NULL); } -/* This function is called in beforeSleep() in order to unblock ASAP clients - * that are waiting in blocking operations with a short timeout set. */ +/* This function is called in beforeSleep() in order to unblock clients + * that are waiting in blocking operations with a timeout set. */ void clientsHandleTimeout(void) { if (raxSize(server.clients_timeout_table) == 0) return; uint64_t now = mstime(); diff --git a/src/server.h b/src/server.h index 7ccab28ba..c6862051a 100644 --- a/src/server.h +++ b/src/server.h @@ -1067,7 +1067,7 @@ struct redisServer { list *clients_pending_read; /* Client has pending read socket buffers. */ list *slaves, *monitors; /* List of slaves and MONITORs */ client *current_client; /* Current client executing the command. */ - rax *clients_timeout_table; /* Radix tree for clients with short timeout. */ + rax *clients_timeout_table; /* Radix tree for blocked clients timeouts. */ long fixed_time_expire; /* If > 0, expire keys against server.mstime. */ rax *clients_index; /* Active clients dictionary by client ID. */ int clients_paused; /* True if clients are currently paused */ From ab10e0f2fba8f9ee68b8a789d75ae5a1a732da19 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 27 Mar 2020 11:13:38 +0100 Subject: [PATCH 0212/1280] Precise timeouts: cleaup the table on unblock. Now that this mechanism is the sole one used for blocked clients timeouts, it is more wise to cleanup the table when the client unblocks for any reason. We use a flag: CLIENT_IN_TO_TABLE, in order to avoid a radix tree lookup when the client was already removed from the table because we processed it by scanning the radix tree. --- src/blocked.c | 1 + src/server.c | 20 ++++++++++++++++++-- src/server.h | 2 ++ 3 files changed, 21 insertions(+), 2 deletions(-) diff --git a/src/blocked.c b/src/blocked.c index 443daec7f..795985ea1 100644 --- a/src/blocked.c +++ b/src/blocked.c @@ -186,6 +186,7 @@ void unblockClient(client *c) { server.blocked_clients_by_type[c->btype]--; c->flags &= ~CLIENT_BLOCKED; c->btype = BLOCKED_NONE; + removeClientFromTimeoutTable(c); queueClientForReprocessing(c); } diff --git a/src/server.c b/src/server.c index 2624cbecd..cfed2f91b 100644 --- a/src/server.c +++ b/src/server.c @@ -1560,7 +1560,20 @@ void addClientToTimeoutTable(client *c) { uint64_t id = c->id; unsigned char buf[CLIENT_ST_KEYLEN]; encodeTimeoutKey(buf,timeout,id); - raxTryInsert(server.clients_timeout_table,buf,sizeof(buf),NULL,NULL); + if (raxTryInsert(server.clients_timeout_table,buf,sizeof(buf),NULL,NULL)) + c->flags |= CLIENT_IN_TO_TABLE; +} + +/* Remove the client from the table when it is unblocked for reasons + * different than timing out. */ +void removeClientFromTimeoutTable(client *c) { + if (!(c->flags & CLIENT_IN_TO_TABLE)) return; + c->flags &= ~CLIENT_IN_TO_TABLE; + uint64_t timeout = c->bpop.timeout; + uint64_t id = c->id; + unsigned char buf[CLIENT_ST_KEYLEN]; + encodeTimeoutKey(buf,timeout,id); + raxRemove(server.clients_timeout_table,buf,sizeof(buf),NULL); } /* This function is called in beforeSleep() in order to unblock clients @@ -1577,7 +1590,10 @@ void clientsHandleTimeout(void) { decodeTimeoutKey(ri.key,&timeout,&id); if (timeout >= now) break; /* All the timeouts are in the future. */ client *c = lookupClientByID(id); - if (c) checkBlockedClientTimeout(c,now); + if (c) { + c->flags &= ~CLIENT_IN_TO_TABLE; + checkBlockedClientTimeout(c,now); + } raxRemove(server.clients_timeout_table,ri.key,ri.key_len,NULL); raxSeek(&ri,"^",NULL,0); } diff --git a/src/server.h b/src/server.h index c6862051a..a3d91a09e 100644 --- a/src/server.h +++ b/src/server.h @@ -252,6 +252,7 @@ typedef long long ustime_t; /* microsecond time type. */ #define CLIENT_TRACKING_OPTOUT (1ULL<<35) /* Tracking in opt-out mode. */ #define CLIENT_TRACKING_CACHING (1ULL<<36) /* CACHING yes/no was given, depending on optin/optout mode. */ +#define CLIENT_IN_TO_TABLE (1ULL<<37) /* This client is in the timeout table. */ /* Client block type (btype field in client structure) * if CLIENT_BLOCKED flag is set. */ @@ -2138,6 +2139,7 @@ void handleClientsBlockedOnKeys(void); void signalKeyAsReady(redisDb *db, robj *key); void blockForKeys(client *c, int btype, robj **keys, int numkeys, mstime_t timeout, robj *target, streamID *ids); void addClientToTimeoutTable(client *c); +void removeClientFromTimeoutTable(client *c); /* expire.c -- Handling of expired keys */ void activeExpireCycle(int type); From aa3efaa1d27c22e2af4b1eb5bde9ca7c8d36f0ca Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 27 Mar 2020 16:34:45 +0100 Subject: [PATCH 0213/1280] timeout.c created: move client timeouts code there. --- src/Makefile | 2 +- src/blocked.c | 39 ---------- src/server.c | 128 +-------------------------------- src/server.h | 4 ++ src/timeout.c | 192 ++++++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 198 insertions(+), 167 deletions(-) create mode 100644 src/timeout.c diff --git a/src/Makefile b/src/Makefile index bbfb06440..3f982cc8e 100644 --- a/src/Makefile +++ b/src/Makefile @@ -206,7 +206,7 @@ endif REDIS_SERVER_NAME=redis-server REDIS_SENTINEL_NAME=redis-sentinel -REDIS_SERVER_OBJ=adlist.o quicklist.o ae.o anet.o dict.o server.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o cluster.o crc16.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o memtest.o crc64.o bitops.o sentinel.o notify.o setproctitle.o blocked.o hyperloglog.o latency.o sparkline.o redis-check-rdb.o redis-check-aof.o geo.o lazyfree.o module.o evict.o expire.o geohash.o geohash_helper.o childinfo.o defrag.o siphash.o rax.o t_stream.o listpack.o localtime.o lolwut.o lolwut5.o lolwut6.o acl.o gopher.o tracking.o connection.o tls.o sha256.o +REDIS_SERVER_OBJ=adlist.o quicklist.o ae.o anet.o dict.o server.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o cluster.o crc16.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o memtest.o crc64.o bitops.o sentinel.o notify.o setproctitle.o blocked.o hyperloglog.o latency.o sparkline.o redis-check-rdb.o redis-check-aof.o geo.o lazyfree.o module.o evict.o expire.o geohash.o geohash_helper.o childinfo.o defrag.o siphash.o rax.o t_stream.o listpack.o localtime.o lolwut.o lolwut5.o lolwut6.o acl.o gopher.o tracking.o connection.o tls.o sha256.o timeout.o REDIS_CLI_NAME=redis-cli REDIS_CLI_OBJ=anet.o adlist.o dict.o redis-cli.o zmalloc.o release.o ae.o crc64.o siphash.o crc16.o REDIS_BENCHMARK_NAME=redis-benchmark diff --git a/src/blocked.c b/src/blocked.c index 795985ea1..e3a803ae3 100644 --- a/src/blocked.c +++ b/src/blocked.c @@ -31,9 +31,6 @@ * * API: * - * getTimeoutFromObjectOrReply() is just an utility function to parse a - * timeout argument since blocking operations usually require a timeout. - * * blockClient() set the CLIENT_BLOCKED flag in the client, and set the * specified block type 'btype' filed to one of BLOCKED_* macros. * @@ -67,42 +64,6 @@ int serveClientBlockedOnList(client *receiver, robj *key, robj *dstkey, redisDb *db, robj *value, int where); -/* Get a timeout value from an object and store it into 'timeout'. - * The final timeout is always stored as milliseconds as a time where the - * timeout will expire, however the parsing is performed according to - * the 'unit' that can be seconds or milliseconds. - * - * Note that if the timeout is zero (usually from the point of view of - * commands API this means no timeout) the value stored into 'timeout' - * is zero. */ -int getTimeoutFromObjectOrReply(client *c, robj *object, mstime_t *timeout, int unit) { - long long tval; - long double ftval; - - if (unit == UNIT_SECONDS) { - if (getLongDoubleFromObjectOrReply(c,object,&ftval, - "timeout is not an float or out of range") != C_OK) - return C_ERR; - tval = (long long) (ftval * 1000.0); - } else { - if (getLongLongFromObjectOrReply(c,object,&tval, - "timeout is not an integer or out of range") != C_OK) - return C_ERR; - } - - if (tval < 0) { - addReplyError(c,"timeout is negative"); - return C_ERR; - } - - if (tval > 0) { - tval += mstime(); - } - *timeout = tval; - - return C_OK; -} - /* Block a client for the specific operation type. Once the CLIENT_BLOCKED * flag is set client query buffer is not longer processed, but accumulated, * and will be processed when the client is unblocked. */ diff --git a/src/server.c b/src/server.c index cfed2f91b..c1f7436a1 100644 --- a/src/server.c +++ b/src/server.c @@ -1473,132 +1473,6 @@ int allPersistenceDisabled(void) { return server.saveparamslen == 0 && server.aof_state == AOF_OFF; } -/* ========================== Clients timeouts ============================= */ - -/* Check if this blocked client timedout (does nothing if the client is - * not blocked right now). If so send a reply, unblock it, and return 1. - * Otherwise 0 is returned and no operation is performed. */ -int checkBlockedClientTimeout(client *c, mstime_t now) { - if (c->flags & CLIENT_BLOCKED && - c->bpop.timeout != 0 - && c->bpop.timeout < now) - { - /* Handle blocking operation specific timeout. */ - replyToBlockedClientTimedOut(c); - unblockClient(c); - return 1; - } else { - return 0; - } -} - -/* Check for timeouts. Returns non-zero if the client was terminated. - * The function gets the current time in milliseconds as argument since - * it gets called multiple times in a loop, so calling gettimeofday() for - * each iteration would be costly without any actual gain. */ -int clientsCronHandleTimeout(client *c, mstime_t now_ms) { - time_t now = now_ms/1000; - - if (server.maxidletime && - /* This handles the idle clients connection timeout if set. */ - !(c->flags & CLIENT_SLAVE) && /* No timeout for slaves and monitors */ - !(c->flags & CLIENT_MASTER) && /* No timeout for masters */ - !(c->flags & CLIENT_BLOCKED) && /* No timeout for BLPOP */ - !(c->flags & CLIENT_PUBSUB) && /* No timeout for Pub/Sub clients */ - (now - c->lastinteraction > server.maxidletime)) - { - serverLog(LL_VERBOSE,"Closing idle client"); - freeClient(c); - return 1; - } else if (c->flags & CLIENT_BLOCKED) { - /* Cluster: handle unblock & redirect of clients blocked - * into keys no longer served by this server. */ - if (server.cluster_enabled) { - if (clusterRedirectBlockedClientIfNeeded(c)) - unblockClient(c); - } - } - return 0; -} - -/* For blocked clients timeouts we populate a radix tree of 128 bit keys - * composed as such: - * - * [8 byte big endian expire time]+[8 byte client ID] - * - * We don't do any cleanup in the Radix tree: when we run the clients that - * reached the timeout already, if they are no longer existing or no longer - * blocked with such timeout, we just go forward. - * - * Every time a client blocks with a timeout, we add the client in - * the tree. In beforeSleep() we call clientsHandleTimeout() to run - * the tree and unblock the clients. */ - -#define CLIENT_ST_KEYLEN 16 /* 8 bytes mstime + 8 bytes client ID. */ - -/* Given client ID and timeout, write the resulting radix tree key in buf. */ -void encodeTimeoutKey(unsigned char *buf, uint64_t timeout, uint64_t id) { - timeout = htonu64(timeout); - memcpy(buf,&timeout,sizeof(timeout)); - memcpy(buf+8,&id,sizeof(id)); -} - -/* Given a key encoded with encodeTimeoutKey(), resolve the fields and write - * the timeout into *toptr and the client ID into *idptr. */ -void decodeTimeoutKey(unsigned char *buf, uint64_t *toptr, uint64_t *idptr) { - memcpy(toptr,buf,sizeof(*toptr)); - *toptr = ntohu64(*toptr); - memcpy(idptr,buf+8,sizeof(*idptr)); -} - -/* Add the specified client id / timeout as a key in the radix tree we use - * to handle blocked clients timeouts. The client is not added to the list - * if its timeout is zero (block forever). */ -void addClientToTimeoutTable(client *c) { - if (c->bpop.timeout == 0) return; - uint64_t timeout = c->bpop.timeout; - uint64_t id = c->id; - unsigned char buf[CLIENT_ST_KEYLEN]; - encodeTimeoutKey(buf,timeout,id); - if (raxTryInsert(server.clients_timeout_table,buf,sizeof(buf),NULL,NULL)) - c->flags |= CLIENT_IN_TO_TABLE; -} - -/* Remove the client from the table when it is unblocked for reasons - * different than timing out. */ -void removeClientFromTimeoutTable(client *c) { - if (!(c->flags & CLIENT_IN_TO_TABLE)) return; - c->flags &= ~CLIENT_IN_TO_TABLE; - uint64_t timeout = c->bpop.timeout; - uint64_t id = c->id; - unsigned char buf[CLIENT_ST_KEYLEN]; - encodeTimeoutKey(buf,timeout,id); - raxRemove(server.clients_timeout_table,buf,sizeof(buf),NULL); -} - -/* This function is called in beforeSleep() in order to unblock clients - * that are waiting in blocking operations with a timeout set. */ -void clientsHandleTimeout(void) { - if (raxSize(server.clients_timeout_table) == 0) return; - uint64_t now = mstime(); - raxIterator ri; - raxStart(&ri,server.clients_timeout_table); - raxSeek(&ri,"^",NULL,0); - - while(raxNext(&ri)) { - uint64_t id, timeout; - decodeTimeoutKey(ri.key,&timeout,&id); - if (timeout >= now) break; /* All the timeouts are in the future. */ - client *c = lookupClientByID(id); - if (c) { - c->flags &= ~CLIENT_IN_TO_TABLE; - checkBlockedClientTimeout(c,now); - } - raxRemove(server.clients_timeout_table,ri.key,ri.key_len,NULL); - raxSeek(&ri,"^",NULL,0); - } -} - /* ======================= Cron: called every 100 ms ======================== */ /* Add a sample to the operations per second array of samples. */ @@ -2183,7 +2057,7 @@ void beforeSleep(struct aeEventLoop *eventLoop) { UNUSED(eventLoop); /* Handle precise timeouts of blocked clients. */ - clientsHandleTimeout(); + handleBlockedClientsTimeout(); /* We should handle pending reads clients ASAP after event loop. */ handleClientsWithPendingReadsUsingThreads(); diff --git a/src/server.h b/src/server.h index a3d91a09e..691f47c48 100644 --- a/src/server.h +++ b/src/server.h @@ -2138,8 +2138,12 @@ void disconnectAllBlockedClients(void); void handleClientsBlockedOnKeys(void); void signalKeyAsReady(redisDb *db, robj *key); void blockForKeys(client *c, int btype, robj **keys, int numkeys, mstime_t timeout, robj *target, streamID *ids); + +/* timeout.c -- Blocked clients timeout and connections timeout. */ void addClientToTimeoutTable(client *c); void removeClientFromTimeoutTable(client *c); +void handleBlockedClientsTimeout(void); +int clientsCronHandleTimeout(client *c, mstime_t now_ms); /* expire.c -- Handling of expired keys */ void activeExpireCycle(int type); diff --git a/src/timeout.c b/src/timeout.c new file mode 100644 index 000000000..ea2032e2a --- /dev/null +++ b/src/timeout.c @@ -0,0 +1,192 @@ +/* Copyright (c) 2009-2020, Salvatore Sanfilippo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include "server.h" +#include "cluster.h" + +/* ========================== Clients timeouts ============================= */ + +/* Check if this blocked client timedout (does nothing if the client is + * not blocked right now). If so send a reply, unblock it, and return 1. + * Otherwise 0 is returned and no operation is performed. */ +int checkBlockedClientTimeout(client *c, mstime_t now) { + if (c->flags & CLIENT_BLOCKED && + c->bpop.timeout != 0 + && c->bpop.timeout < now) + { + /* Handle blocking operation specific timeout. */ + replyToBlockedClientTimedOut(c); + unblockClient(c); + return 1; + } else { + return 0; + } +} + +/* Check for timeouts. Returns non-zero if the client was terminated. + * The function gets the current time in milliseconds as argument since + * it gets called multiple times in a loop, so calling gettimeofday() for + * each iteration would be costly without any actual gain. */ +int clientsCronHandleTimeout(client *c, mstime_t now_ms) { + time_t now = now_ms/1000; + + if (server.maxidletime && + /* This handles the idle clients connection timeout if set. */ + !(c->flags & CLIENT_SLAVE) && /* No timeout for slaves and monitors */ + !(c->flags & CLIENT_MASTER) && /* No timeout for masters */ + !(c->flags & CLIENT_BLOCKED) && /* No timeout for BLPOP */ + !(c->flags & CLIENT_PUBSUB) && /* No timeout for Pub/Sub clients */ + (now - c->lastinteraction > server.maxidletime)) + { + serverLog(LL_VERBOSE,"Closing idle client"); + freeClient(c); + return 1; + } else if (c->flags & CLIENT_BLOCKED) { + /* Cluster: handle unblock & redirect of clients blocked + * into keys no longer served by this server. */ + if (server.cluster_enabled) { + if (clusterRedirectBlockedClientIfNeeded(c)) + unblockClient(c); + } + } + return 0; +} + +/* For blocked clients timeouts we populate a radix tree of 128 bit keys + * composed as such: + * + * [8 byte big endian expire time]+[8 byte client ID] + * + * We don't do any cleanup in the Radix tree: when we run the clients that + * reached the timeout already, if they are no longer existing or no longer + * blocked with such timeout, we just go forward. + * + * Every time a client blocks with a timeout, we add the client in + * the tree. In beforeSleep() we call handleBlockedClientsTimeout() to run + * the tree and unblock the clients. */ + +#define CLIENT_ST_KEYLEN 16 /* 8 bytes mstime + 8 bytes client ID. */ + +/* Given client ID and timeout, write the resulting radix tree key in buf. */ +void encodeTimeoutKey(unsigned char *buf, uint64_t timeout, uint64_t id) { + timeout = htonu64(timeout); + memcpy(buf,&timeout,sizeof(timeout)); + memcpy(buf+8,&id,sizeof(id)); +} + +/* Given a key encoded with encodeTimeoutKey(), resolve the fields and write + * the timeout into *toptr and the client ID into *idptr. */ +void decodeTimeoutKey(unsigned char *buf, uint64_t *toptr, uint64_t *idptr) { + memcpy(toptr,buf,sizeof(*toptr)); + *toptr = ntohu64(*toptr); + memcpy(idptr,buf+8,sizeof(*idptr)); +} + +/* Add the specified client id / timeout as a key in the radix tree we use + * to handle blocked clients timeouts. The client is not added to the list + * if its timeout is zero (block forever). */ +void addClientToTimeoutTable(client *c) { + if (c->bpop.timeout == 0) return; + uint64_t timeout = c->bpop.timeout; + uint64_t id = c->id; + unsigned char buf[CLIENT_ST_KEYLEN]; + encodeTimeoutKey(buf,timeout,id); + if (raxTryInsert(server.clients_timeout_table,buf,sizeof(buf),NULL,NULL)) + c->flags |= CLIENT_IN_TO_TABLE; +} + +/* Remove the client from the table when it is unblocked for reasons + * different than timing out. */ +void removeClientFromTimeoutTable(client *c) { + if (!(c->flags & CLIENT_IN_TO_TABLE)) return; + c->flags &= ~CLIENT_IN_TO_TABLE; + uint64_t timeout = c->bpop.timeout; + uint64_t id = c->id; + unsigned char buf[CLIENT_ST_KEYLEN]; + encodeTimeoutKey(buf,timeout,id); + raxRemove(server.clients_timeout_table,buf,sizeof(buf),NULL); +} + +/* This function is called in beforeSleep() in order to unblock clients + * that are waiting in blocking operations with a timeout set. */ +void handleBlockedClientsTimeout(void) { + if (raxSize(server.clients_timeout_table) == 0) return; + uint64_t now = mstime(); + raxIterator ri; + raxStart(&ri,server.clients_timeout_table); + raxSeek(&ri,"^",NULL,0); + + while(raxNext(&ri)) { + uint64_t id, timeout; + decodeTimeoutKey(ri.key,&timeout,&id); + if (timeout >= now) break; /* All the timeouts are in the future. */ + client *c = lookupClientByID(id); + if (c) { + c->flags &= ~CLIENT_IN_TO_TABLE; + checkBlockedClientTimeout(c,now); + } + raxRemove(server.clients_timeout_table,ri.key,ri.key_len,NULL); + raxSeek(&ri,"^",NULL,0); + } +} + +/* Get a timeout value from an object and store it into 'timeout'. + * The final timeout is always stored as milliseconds as a time where the + * timeout will expire, however the parsing is performed according to + * the 'unit' that can be seconds or milliseconds. + * + * Note that if the timeout is zero (usually from the point of view of + * commands API this means no timeout) the value stored into 'timeout' + * is zero. */ +int getTimeoutFromObjectOrReply(client *c, robj *object, mstime_t *timeout, int unit) { + long long tval; + long double ftval; + + if (unit == UNIT_SECONDS) { + if (getLongDoubleFromObjectOrReply(c,object,&ftval, + "timeout is not an float or out of range") != C_OK) + return C_ERR; + tval = (long long) (ftval * 1000.0); + } else { + if (getLongLongFromObjectOrReply(c,object,&tval, + "timeout is not an integer or out of range") != C_OK) + return C_ERR; + } + + if (tval < 0) { + addReplyError(c,"timeout is negative"); + return C_ERR; + } + + if (tval > 0) { + tval += mstime(); + } + *timeout = tval; + + return C_OK; +} From 9a1be653adcaf2a8cec4b2386c45b7d388a205eb Mon Sep 17 00:00:00 2001 From: "zhaozhao.zz" Date: Sat, 28 Mar 2020 20:59:01 +0800 Subject: [PATCH 0214/1280] PSYNC2: reset backlog_idx and master_repl_offset correctly --- src/replication.c | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/replication.c b/src/replication.c index 20f9a2129..3e9910374 100644 --- a/src/replication.c +++ b/src/replication.c @@ -2754,11 +2754,16 @@ void replicationCacheMasterUsingMyself(void) { server.master_repl_offset, delta); server.master_initial_offset = server.master_repl_meaningful_offset; - server.repl_backlog_histlen -= delta; - server.repl_backlog_idx = - (server.repl_backlog_idx + (server.repl_backlog_size - delta)) % - server.repl_backlog_size; - if (server.repl_backlog_histlen < 0) server.repl_backlog_histlen = 0; + server.master_repl_offset = server.master_repl_meaningful_offset; + if (server.repl_backlog_histlen <= delta) { + server.repl_backlog_histlen = 0; + server.repl_backlog_idx = 0; + } else { + server.repl_backlog_histlen -= delta; + server.repl_backlog_idx = + (server.repl_backlog_idx + (server.repl_backlog_size - delta)) % + server.repl_backlog_size; + } } /* The master client we create can be set to any DBID, because From 5120922a86f92166a0c772353871ee51e05f4ae9 Mon Sep 17 00:00:00 2001 From: OMG-By <504094596@qq.com> Date: Sun, 29 Mar 2020 00:04:59 +0800 Subject: [PATCH 0215/1280] fix: dict.c->dictResize()->minimal type --- src/dict.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dict.c b/src/dict.c index 106467ef7..dec15771d 100644 --- a/src/dict.c +++ b/src/dict.c @@ -134,7 +134,7 @@ int _dictInit(dict *d, dictType *type, * but with the invariant of a USED/BUCKETS ratio near to <= 1 */ int dictResize(dict *d) { - int minimal; + unsigned long minimal; if (!dict_can_resize || dictIsRehashing(d)) return DICT_ERR; minimal = d->ht[0].used; From 5751ba1f2cdae190ae57d84de8c28223a7e968a7 Mon Sep 17 00:00:00 2001 From: Guy Benoish Date: Sun, 29 Mar 2020 17:50:42 +0300 Subject: [PATCH 0216/1280] PERSIST should notify a keyspace event --- src/expire.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/expire.c b/src/expire.c index 5aff72ee0..c102a01ff 100644 --- a/src/expire.c +++ b/src/expire.c @@ -590,6 +590,7 @@ void pttlCommand(client *c) { void persistCommand(client *c) { if (lookupKeyWrite(c->db,c->argv[1])) { if (removeExpire(c->db,c->argv[1])) { + notifyKeyspaceEvent(NOTIFY_GENERIC,"persist",c->argv[1],c->db->id); addReply(c,shared.cone); server.dirty++; } else { From a5533e0ad85d2d19699a0d74fe3789e00f8cc0c5 Mon Sep 17 00:00:00 2001 From: hwware Date: Sun, 29 Mar 2020 23:06:50 -0400 Subject: [PATCH 0217/1280] add check for not providing both optin optout flag --- src/networking.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/networking.c b/src/networking.c index a550e4040..bad3e2c25 100644 --- a/src/networking.c +++ b/src/networking.c @@ -2314,6 +2314,14 @@ NULL return; } + if ((options & CLIENT_TRACKING_OPTIN) && (options & CLIENT_TRACKING_OPTOUT)) + { + addReplyError(c, + "You can't specify both OPTIN mode and OPTOUT mode"); + zfree(prefix); + return; + } + enableTracking(c,redir,options,prefix,numprefix); } else if (!strcasecmp(c->argv[2]->ptr,"off")) { disableTracking(c); From 2d1a2896f874b473b188c593780d18978bcf4d9a Mon Sep 17 00:00:00 2001 From: hwware Date: Sun, 29 Mar 2020 23:20:54 -0400 Subject: [PATCH 0218/1280] add check for not switching between optin optout mode directly --- src/networking.c | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/networking.c b/src/networking.c index bad3e2c25..b6ab8eb5d 100644 --- a/src/networking.c +++ b/src/networking.c @@ -2314,7 +2314,7 @@ NULL return; } - if ((options & CLIENT_TRACKING_OPTIN) && (options & CLIENT_TRACKING_OPTOUT)) + if (options & CLIENT_TRACKING_OPTIN && options & CLIENT_TRACKING_OPTOUT) { addReplyError(c, "You can't specify both OPTIN mode and OPTOUT mode"); @@ -2322,6 +2322,17 @@ NULL return; } + if ((options & CLIENT_TRACKING_OPTIN && c->flags & CLIENT_TRACKING_OPTOUT) || + (options & CLIENT_TRACKING_OPTOUT && c->flags & CLIENT_TRACKING_OPTIN)) + { + addReplyError(c, + "You can't switch OPTIN/OPTOUT mode before disabling " + "tracking for this client, and then re-enabling it with " + "a different mode."); + zfree(prefix); + return; + } + enableTracking(c,redir,options,prefix,numprefix); } else if (!strcasecmp(c->argv[2]->ptr,"off")) { disableTracking(c); From 5dba96dc7f7eb6d01fab4ae769aa77bf8abb5f84 Mon Sep 17 00:00:00 2001 From: Guy Benoish Date: Mon, 30 Mar 2020 10:52:59 +0300 Subject: [PATCH 0219/1280] streamReplyWithRange: Redundant XSETIDs to replica propagate_last_id is declared outside of the loop but used only from within the loop. Once it's '1' it will never go back to '0' and will replicate XSETID even for IDs that don't actually change the last_id. While not a serious bug (XSETID always used group->last_id so there's no risk), it does causes redundant traffic between master and its replicas --- src/t_stream.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/t_stream.c b/src/t_stream.c index 557d1d642..00d1cbf1c 100644 --- a/src/t_stream.c +++ b/src/t_stream.c @@ -935,7 +935,6 @@ size_t streamReplyWithRange(client *c, stream *s, streamID *start, streamID *end streamIterator si; int64_t numfields; streamID id; - int propagate_last_id = 0; /* If the client is asking for some history, we serve it using a * different function, so that we return entries *solely* from its @@ -951,6 +950,8 @@ size_t streamReplyWithRange(client *c, stream *s, streamID *start, streamID *end arraylen_ptr = addReplyDeferredLen(c); streamIteratorStart(&si,s,start,end,rev); while(streamIteratorGetID(&si,&id,&numfields)) { + int propagate_last_id = 0; + /* Update the group last_id if needed. */ if (group && streamCompareID(&id,&group->last_id) > 0) { group->last_id = id; From 6dceb0140ea8c76d4779977b58d96206ff9c9dd0 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 30 Mar 2020 15:22:55 +0200 Subject: [PATCH 0220/1280] Precise timeouts: reference client pointer directly. --- src/timeout.c | 29 +++++++++++++---------------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/src/timeout.c b/src/timeout.c index ea2032e2a..bb5999418 100644 --- a/src/timeout.c +++ b/src/timeout.c @@ -93,18 +93,19 @@ int clientsCronHandleTimeout(client *c, mstime_t now_ms) { #define CLIENT_ST_KEYLEN 16 /* 8 bytes mstime + 8 bytes client ID. */ /* Given client ID and timeout, write the resulting radix tree key in buf. */ -void encodeTimeoutKey(unsigned char *buf, uint64_t timeout, uint64_t id) { +void encodeTimeoutKey(unsigned char *buf, uint64_t timeout, client *c) { timeout = htonu64(timeout); memcpy(buf,&timeout,sizeof(timeout)); - memcpy(buf+8,&id,sizeof(id)); + memcpy(buf+8,&c,sizeof(c)); + if (sizeof(c) == 4) memset(buf+12,0,4); /* Zero padding for 32bit target. */ } /* Given a key encoded with encodeTimeoutKey(), resolve the fields and write - * the timeout into *toptr and the client ID into *idptr. */ -void decodeTimeoutKey(unsigned char *buf, uint64_t *toptr, uint64_t *idptr) { + * the timeout into *toptr and the client pointer into *cptr. */ +void decodeTimeoutKey(unsigned char *buf, uint64_t *toptr, client **cptr) { memcpy(toptr,buf,sizeof(*toptr)); *toptr = ntohu64(*toptr); - memcpy(idptr,buf+8,sizeof(*idptr)); + memcpy(cptr,buf+8,sizeof(*cptr)); } /* Add the specified client id / timeout as a key in the radix tree we use @@ -113,9 +114,8 @@ void decodeTimeoutKey(unsigned char *buf, uint64_t *toptr, uint64_t *idptr) { void addClientToTimeoutTable(client *c) { if (c->bpop.timeout == 0) return; uint64_t timeout = c->bpop.timeout; - uint64_t id = c->id; unsigned char buf[CLIENT_ST_KEYLEN]; - encodeTimeoutKey(buf,timeout,id); + encodeTimeoutKey(buf,timeout,c); if (raxTryInsert(server.clients_timeout_table,buf,sizeof(buf),NULL,NULL)) c->flags |= CLIENT_IN_TO_TABLE; } @@ -126,9 +126,8 @@ void removeClientFromTimeoutTable(client *c) { if (!(c->flags & CLIENT_IN_TO_TABLE)) return; c->flags &= ~CLIENT_IN_TO_TABLE; uint64_t timeout = c->bpop.timeout; - uint64_t id = c->id; unsigned char buf[CLIENT_ST_KEYLEN]; - encodeTimeoutKey(buf,timeout,id); + encodeTimeoutKey(buf,timeout,c); raxRemove(server.clients_timeout_table,buf,sizeof(buf),NULL); } @@ -142,14 +141,12 @@ void handleBlockedClientsTimeout(void) { raxSeek(&ri,"^",NULL,0); while(raxNext(&ri)) { - uint64_t id, timeout; - decodeTimeoutKey(ri.key,&timeout,&id); + uint64_t timeout; + client *c; + decodeTimeoutKey(ri.key,&timeout,&c); if (timeout >= now) break; /* All the timeouts are in the future. */ - client *c = lookupClientByID(id); - if (c) { - c->flags &= ~CLIENT_IN_TO_TABLE; - checkBlockedClientTimeout(c,now); - } + c->flags &= ~CLIENT_IN_TO_TABLE; + checkBlockedClientTimeout(c,now); raxRemove(server.clients_timeout_table,ri.key,ri.key_len,NULL); raxSeek(&ri,"^",NULL,0); } From 80bb739ca59e35df5a0c3bee19294504fd67b19e Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 31 Mar 2020 10:27:47 +0200 Subject: [PATCH 0221/1280] Fix RM_Call() stale comment due to cut&paste. --- src/module.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/module.c b/src/module.c index 6f61a5ca8..7d9eb3719 100644 --- a/src/module.c +++ b/src/module.c @@ -3341,9 +3341,7 @@ RedisModuleCallReply *RM_Call(RedisModuleCtx *ctx, const char *cmdname, const ch } call(c,call_flags); - /* Convert the result of the Redis command into a suitable Lua type. - * The first thing we need is to create a single string from the client - * output buffers. */ + /* Convert the result of the Redis command into a module reply. */ sds proto = sdsnewlen(c->buf,c->bufpos); c->bufpos = 0; while(listLength(c->reply)) { From ab89ab5173b94b203c0e87d43e253e3ebb10c170 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 31 Mar 2020 11:00:45 +0200 Subject: [PATCH 0222/1280] Fix module commands propagation double MULTI bug. b512cb40 introduced automatic wrapping of MULTI/EXEC for the alsoPropagate API. However this collides with the built-in mechanism already present in module.c. To avoid complex changes near Redis 6 GA this commit introduces the ability to exclude call() MUTLI/EXEC wrapping for also propagate in order to continue to use the old code paths in module.c. --- src/module.c | 7 +------ src/server.c | 8 ++++++-- src/server.h | 2 ++ tests/modules/propagate.c | 16 ++++++++++++++++ 4 files changed, 25 insertions(+), 8 deletions(-) diff --git a/src/module.c b/src/module.c index 7d9eb3719..61dc25169 100644 --- a/src/module.c +++ b/src/module.c @@ -3326,13 +3326,8 @@ RedisModuleCallReply *RM_Call(RedisModuleCtx *ctx, const char *cmdname, const ch * a Lua script in the context of AOF and slaves. */ if (replicate) moduleReplicateMultiIfNeeded(ctx); - if (ctx->client->flags & CLIENT_MULTI || - ctx->flags & REDISMODULE_CTX_MULTI_EMITTED) { - c->flags |= CLIENT_MULTI; - } - /* Run the command */ - int call_flags = CMD_CALL_SLOWLOG | CMD_CALL_STATS; + int call_flags = CMD_CALL_SLOWLOG | CMD_CALL_STATS | CMD_CALL_NOWRAP; if (replicate) { if (!(flags & REDISMODULE_ARGV_NO_AOF)) call_flags |= CMD_CALL_PROPAGATE_AOF; diff --git a/src/server.c b/src/server.c index c1f7436a1..93bafbae8 100644 --- a/src/server.c +++ b/src/server.c @@ -3266,11 +3266,15 @@ void call(client *c, int flags) { int multi_emitted = 0; /* Wrap the commands in server.also_propagate array, * but don't wrap it if we are already in MULIT context, - * in case the nested MULIT/EXEC. + * in case the nested MULTI/EXEC. * * And if the array contains only one command, no need to * wrap it, since the single command is atomic. */ - if (server.also_propagate.numops > 1 && !(c->flags & CLIENT_MULTI)) { + if (server.also_propagate.numops > 1 && + !(c->cmd->flags & CMD_MODULE) && + !(c->flags & CLIENT_MULTI) && + !(flags & CMD_CALL_NOWRAP)) + { execCommandPropagateMulti(c); multi_emitted = 1; } diff --git a/src/server.h b/src/server.h index 691f47c48..d5be5ed29 100644 --- a/src/server.h +++ b/src/server.h @@ -395,6 +395,8 @@ typedef long long ustime_t; /* microsecond time type. */ #define CMD_CALL_PROPAGATE_REPL (1<<3) #define CMD_CALL_PROPAGATE (CMD_CALL_PROPAGATE_AOF|CMD_CALL_PROPAGATE_REPL) #define CMD_CALL_FULL (CMD_CALL_SLOWLOG | CMD_CALL_STATS | CMD_CALL_PROPAGATE) +#define CMD_CALL_NOWRAP (1<<4) /* Don't wrap also propagate array into + MULTI/EXEC: the caller will handle it. */ /* Command propagation flags, see propagate() function */ #define PROPAGATE_NONE 0 diff --git a/tests/modules/propagate.c b/tests/modules/propagate.c index f83af1799..2571ee43d 100644 --- a/tests/modules/propagate.c +++ b/tests/modules/propagate.c @@ -89,6 +89,16 @@ int propagateTestCommand(RedisModuleCtx *ctx, RedisModuleString **argv, int argc return REDISMODULE_OK; } +int propagateTest2Command(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) +{ + REDISMODULE_NOT_USED(argv); + REDISMODULE_NOT_USED(argc); + + RedisModule_Replicate(ctx,"INCR","c","counter"); + RedisModule_ReplyWithSimpleString(ctx,"OK"); + return REDISMODULE_OK; +} + int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { REDISMODULE_NOT_USED(argv); REDISMODULE_NOT_USED(argc); @@ -100,5 +110,11 @@ int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) propagateTestCommand, "",1,1,1) == REDISMODULE_ERR) return REDISMODULE_ERR; + + if (RedisModule_CreateCommand(ctx,"propagate-test-2", + propagateTest2Command, + "",1,1,1) == REDISMODULE_ERR) + return REDISMODULE_ERR; + return REDISMODULE_OK; } From 55a751597a12354755732b45d9187c5cdd9214b1 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 31 Mar 2020 12:04:06 +0200 Subject: [PATCH 0223/1280] Modify the propagate unit test to show more cases. --- tests/modules/propagate.c | 32 ++++++++++++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/tests/modules/propagate.c b/tests/modules/propagate.c index 2571ee43d..13277b19d 100644 --- a/tests/modules/propagate.c +++ b/tests/modules/propagate.c @@ -64,7 +64,8 @@ void *threadMain(void *arg) { RedisModule_SelectDb(ctx,9); /* Tests ran in database number 9. */ for (int i = 0; i < 10; i++) { RedisModule_ThreadSafeContextLock(ctx); - RedisModule_Replicate(ctx,"INCR","c","thread"); + RedisModule_Replicate(ctx,"INCR","c","a-from-thread"); + RedisModule_Replicate(ctx,"INCR","c","b-from-thread"); RedisModule_ThreadSafeContextUnlock(ctx); } RedisModule_FreeThreadSafeContext(ctx); @@ -94,7 +95,29 @@ int propagateTest2Command(RedisModuleCtx *ctx, RedisModuleString **argv, int arg REDISMODULE_NOT_USED(argv); REDISMODULE_NOT_USED(argc); - RedisModule_Replicate(ctx,"INCR","c","counter"); + /* Replicate two commands to test MULTI/EXEC wrapping. */ + RedisModule_Replicate(ctx,"INCR","c","counter-1"); + RedisModule_Replicate(ctx,"INCR","c","counter-2"); + RedisModule_ReplyWithSimpleString(ctx,"OK"); + return REDISMODULE_OK; +} + +int propagateTest3Command(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) +{ + REDISMODULE_NOT_USED(argv); + REDISMODULE_NOT_USED(argc); + RedisModuleCallReply *reply; + + /* This test mixes multiple propagation systems. */ + reply = RedisModule_Call(ctx, "INCR", "c!", "using-call"); + RedisModule_FreeCallReply(reply); + + RedisModule_Replicate(ctx,"INCR","c","counter-1"); + RedisModule_Replicate(ctx,"INCR","c","counter-2"); + + reply = RedisModule_Call(ctx, "INCR", "c!", "after-call"); + RedisModule_FreeCallReply(reply); + RedisModule_ReplyWithSimpleString(ctx,"OK"); return REDISMODULE_OK; } @@ -116,5 +139,10 @@ int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) "",1,1,1) == REDISMODULE_ERR) return REDISMODULE_ERR; + if (RedisModule_CreateCommand(ctx,"propagate-test-3", + propagateTest3Command, + "",1,1,1) == REDISMODULE_ERR) + return REDISMODULE_ERR; + return REDISMODULE_OK; } From 0b7cdb9dfb2cbea51265a42af822689620d7a490 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 31 Mar 2020 12:09:38 +0200 Subject: [PATCH 0224/1280] Fix the propagate Tcl test after module changes. --- tests/unit/moduleapi/propagate.tcl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/moduleapi/propagate.tcl b/tests/unit/moduleapi/propagate.tcl index 71307ce33..73f795c71 100644 --- a/tests/unit/moduleapi/propagate.tcl +++ b/tests/unit/moduleapi/propagate.tcl @@ -20,7 +20,7 @@ tags "modules" { wait_for_condition 5000 10 { ([$replica get timer] eq "10") && \ - ([$replica get thread] eq "10") + ([$replica get a-from-thread] eq "10") } else { fail "The two counters don't match the expected value." } From fd914fdd527135cfb5086c3a94c8a66cd521d902 Mon Sep 17 00:00:00 2001 From: Guy Benoish Date: Sun, 29 Mar 2020 13:08:21 +0300 Subject: [PATCH 0225/1280] Modules: Test MULTI/EXEC replication of RM_Replicate Makse sure call() doesn't wrap replicated commands with a redundant MULTI/EXEC Other, unrelated changes: 1. Formatting compiler warning in INFO CLIENTS 2. Use CLIENT_ID_AOF instead of UINT64_MAX --- src/aof.c | 2 +- src/networking.c | 9 +++++--- src/redismodule.h | 2 +- src/server.c | 11 ++++++---- src/server.h | 1 + tests/unit/moduleapi/propagate.tcl | 33 ++++++++++++++++++++++++++++++ 6 files changed, 49 insertions(+), 9 deletions(-) diff --git a/src/aof.c b/src/aof.c index 6bb239252..301a40848 100644 --- a/src/aof.c +++ b/src/aof.c @@ -831,7 +831,7 @@ int loadAppendOnlyFile(char *filename) { if (cmd == server.multiCommand) valid_before_multi = valid_up_to; /* Run the command in the context of a fake client */ - fakeClient->cmd = cmd; + fakeClient->cmd = fakeClient->lastcmd = cmd; if (fakeClient->flags & CLIENT_MULTI && fakeClient->cmd->proc != execCommand) { diff --git a/src/networking.c b/src/networking.c index a550e4040..fcaa164a9 100644 --- a/src/networking.c +++ b/src/networking.c @@ -373,13 +373,16 @@ void addReplyErrorLength(client *c, const char *s, size_t len) { * will produce an error. However it is useful to log such events since * they are rare and may hint at errors in a script or a bug in Redis. */ int ctype = getClientType(c); - if (ctype == CLIENT_TYPE_MASTER || ctype == CLIENT_TYPE_SLAVE) { - char* to = ctype == CLIENT_TYPE_MASTER? "master": "replica"; - char* from = ctype == CLIENT_TYPE_MASTER? "replica": "master"; + if (ctype == CLIENT_TYPE_MASTER || ctype == CLIENT_TYPE_SLAVE || c->id == CLIENT_ID_AOF) { + char* to = c->id == CLIENT_ID_AOF ? "AOF-client" : + ctype == CLIENT_TYPE_MASTER ? "master" : "replica"; + char* from = c->id == CLIENT_ID_AOF ? "server" : + ctype == CLIENT_TYPE_MASTER ? "replica" : "master"; char *cmdname = c->lastcmd ? c->lastcmd->name : ""; serverLog(LL_WARNING,"== CRITICAL == This %s is sending an error " "to its %s: '%s' after processing the command " "'%s'", from, to, s, cmdname); + server.stat_unexpected_error_replies++; } } diff --git a/src/redismodule.h b/src/redismodule.h index a43443f13..d26c41456 100644 --- a/src/redismodule.h +++ b/src/redismodule.h @@ -673,7 +673,7 @@ int REDISMODULE_API_FUNC(RedisModule_AuthenticateClientWithUser)(RedisModuleCtx void REDISMODULE_API_FUNC(RedisModule_DeauthenticateAndCloseClient)(RedisModuleCtx *ctx, uint64_t client_id); #endif -#define RedisModule_IsAOFClient(id) ((id) == UINT64_MAX) +#define RedisModule_IsAOFClient(id) ((id) == CLIENT_ID_AOF) /* This is included inline inside each Redis module. */ static int RedisModule_Init(RedisModuleCtx *ctx, const char *name, int ver, int apiver) __attribute__((unused)); diff --git a/src/server.c b/src/server.c index 93bafbae8..33f875ba9 100644 --- a/src/server.c +++ b/src/server.c @@ -2661,6 +2661,7 @@ void resetServerStats(void) { } server.stat_net_input_bytes = 0; server.stat_net_output_bytes = 0; + server.stat_unexpected_error_replies = 0; server.aof_delayed_fsync = 0; } @@ -3265,7 +3266,7 @@ void call(client *c, int flags) { if (flags & CMD_CALL_PROPAGATE) { int multi_emitted = 0; /* Wrap the commands in server.also_propagate array, - * but don't wrap it if we are already in MULIT context, + * but don't wrap it if we are already in MULTI context, * in case the nested MULTI/EXEC. * * And if the array contains only one command, no need to @@ -3974,7 +3975,7 @@ sds genRedisInfoString(const char *section) { "client_recent_max_output_buffer:%zu\r\n" "blocked_clients:%d\r\n" "tracking_clients:%d\r\n" - "clients_in_timeout_table:%lld\r\n", + "clients_in_timeout_table:%ld\r\n", listLength(server.clients)-listLength(server.slaves), maxin, maxout, server.blocked_clients, @@ -4229,7 +4230,8 @@ sds genRedisInfoString(const char *section) { "active_defrag_key_hits:%lld\r\n" "active_defrag_key_misses:%lld\r\n" "tracking_total_keys:%lld\r\n" - "tracking_total_items:%lld\r\n", + "tracking_total_items:%lld\r\n" + "unexpected_error_replies:%lld\r\n", server.stat_numconnections, server.stat_numcommands, getInstantaneousMetric(STATS_METRIC_COMMAND), @@ -4258,7 +4260,8 @@ sds genRedisInfoString(const char *section) { server.stat_active_defrag_key_hits, server.stat_active_defrag_key_misses, (unsigned long long) trackingGetTotalKeys(), - (unsigned long long) trackingGetTotalItems()); + (unsigned long long) trackingGetTotalItems(), + server.stat_unexpected_error_replies); } /* Replication */ diff --git a/src/server.h b/src/server.h index d5be5ed29..59cec9352 100644 --- a/src/server.h +++ b/src/server.h @@ -1129,6 +1129,7 @@ struct redisServer { size_t stat_rdb_cow_bytes; /* Copy on write bytes during RDB saving. */ size_t stat_aof_cow_bytes; /* Copy on write bytes during AOF rewrite. */ size_t stat_module_cow_bytes; /* Copy on write bytes during module fork. */ + long long stat_unexpected_error_replies; /* Number of unexpected (aof-loading, replica to master, etc.) error replies */ /* The following two are used to track instantaneous metrics, like * number of operations per second, network traffic. */ struct { diff --git a/tests/unit/moduleapi/propagate.tcl b/tests/unit/moduleapi/propagate.tcl index 73f795c71..aa0f55e5e 100644 --- a/tests/unit/moduleapi/propagate.tcl +++ b/tests/unit/moduleapi/propagate.tcl @@ -24,7 +24,40 @@ tags "modules" { } else { fail "The two counters don't match the expected value." } + + $master propagate-test-2 + $master propagate-test-3 + $master multi + $master propagate-test-2 + $master propagate-test-3 + $master exec + wait_for_ofs_sync $master $replica + + assert_equal [s -1 unexpected_error_replies] 0 } } } } + +tags "modules aof" { + test {Modules RM_Replicate replicates MULTI/EXEC correctly} { + start_server [list overrides [list loadmodule "$testmodule"]] { + # Enable the AOF + r config set appendonly yes + r config set auto-aof-rewrite-percentage 0 ; # Disable auto-rewrite. + waitForBgrewriteaof r + + r propagate-test-2 + r propagate-test-3 + r multi + r propagate-test-2 + r propagate-test-3 + r exec + + # Load the AOF + r debug loadaof + + assert_equal [s 0 unexpected_error_replies] 0 + } + } +} From e5309fea930d3a433430323906e049ecc993a5dc Mon Sep 17 00:00:00 2001 From: Guy Benoish Date: Tue, 31 Mar 2020 17:37:05 +0300 Subject: [PATCH 0226/1280] RENAME can unblock XREADGROUP Other changes: Support stream in serverLogObjectDebugInfo --- src/db.c | 3 ++- src/debug.c | 2 ++ tests/unit/type/stream-cgroups.tcl | 21 +++++++++++++++++++++ 3 files changed, 25 insertions(+), 1 deletion(-) diff --git a/src/db.c b/src/db.c index 04e26c33b..211bb978d 100644 --- a/src/db.c +++ b/src/db.c @@ -182,7 +182,8 @@ void dbAdd(redisDb *db, robj *key, robj *val) { serverAssertWithInfo(NULL,key,retval == DICT_OK); if (val->type == OBJ_LIST || - val->type == OBJ_ZSET) + val->type == OBJ_ZSET || + val->type == OBJ_STREAM) signalKeyAsReady(db, key); if (server.cluster_enabled) slotToKeyAdd(key); } diff --git a/src/debug.c b/src/debug.c index 36af35aec..83e5b6197 100644 --- a/src/debug.c +++ b/src/debug.c @@ -817,6 +817,8 @@ void serverLogObjectDebugInfo(const robj *o) { serverLog(LL_WARNING,"Sorted set size: %d", (int) zsetLength(o)); if (o->encoding == OBJ_ENCODING_SKIPLIST) serverLog(LL_WARNING,"Skiplist level: %d", (int) ((const zset*)o->ptr)->zsl->level); + } else if (o->type == OBJ_STREAM) { + serverLog(LL_WARNING,"Stream size: %d", (int) streamLength(o)); } } diff --git a/tests/unit/type/stream-cgroups.tcl b/tests/unit/type/stream-cgroups.tcl index 6b9a4a9cd..a27e1f582 100644 --- a/tests/unit/type/stream-cgroups.tcl +++ b/tests/unit/type/stream-cgroups.tcl @@ -170,6 +170,27 @@ start_server { assert_error "*NOGROUP*" {$rd read} } + test {RENAME can unblock XREADGROUP with data} { + r del mystream + r XGROUP CREATE mystream mygroup $ MKSTREAM + set rd [redis_deferring_client] + $rd XREADGROUP GROUP mygroup Alice BLOCK 0 STREAMS mystream ">" + r XGROUP CREATE mystream2 mygroup $ MKSTREAM + r XADD mystream2 100 f1 v1 + r RENAME mystream2 mystream + assert_equal "{mystream {{100-0 {f1 v1}}}}" [$rd read] ;# mystream2 had mygroup before RENAME + } + + test {RENAME can unblock XREADGROUP with -NOGROUP} { + r del mystream + r XGROUP CREATE mystream mygroup $ MKSTREAM + set rd [redis_deferring_client] + $rd XREADGROUP GROUP mygroup Alice BLOCK 0 STREAMS mystream ">" + r XADD mystream2 100 f1 v1 + r RENAME mystream2 mystream + assert_error "*NOGROUP*" {$rd read} ;# mystream2 didn't have mygroup before RENAME + } + test {XCLAIM can claim PEL items from another consumer} { # Add 3 items into the stream, and create a consumer group r del mystream From f93573cb6b5afff561ed773d9ac50b43f54db101 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 31 Mar 2020 17:10:09 +0200 Subject: [PATCH 0227/1280] Minor changes to #7037. --- src/networking.c | 17 +++++++++++++---- src/server.c | 2 +- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/src/networking.c b/src/networking.c index fcaa164a9..3c754c376 100644 --- a/src/networking.c +++ b/src/networking.c @@ -374,10 +374,19 @@ void addReplyErrorLength(client *c, const char *s, size_t len) { * they are rare and may hint at errors in a script or a bug in Redis. */ int ctype = getClientType(c); if (ctype == CLIENT_TYPE_MASTER || ctype == CLIENT_TYPE_SLAVE || c->id == CLIENT_ID_AOF) { - char* to = c->id == CLIENT_ID_AOF ? "AOF-client" : - ctype == CLIENT_TYPE_MASTER ? "master" : "replica"; - char* from = c->id == CLIENT_ID_AOF ? "server" : - ctype == CLIENT_TYPE_MASTER ? "replica" : "master"; + char *to, *from; + + if (c->id == CLIENT_ID_AOF) { + to = "AOF-loading-client"; + from = "server"; + } else if (ctype == CLIENT_TYPE_MASTER) { + to = "master"; + from = "replica"; + } else { + to = "replica"; + from = "master"; + } + char *cmdname = c->lastcmd ? c->lastcmd->name : ""; serverLog(LL_WARNING,"== CRITICAL == This %s is sending an error " "to its %s: '%s' after processing the command " diff --git a/src/server.c b/src/server.c index 6f1913e4d..4fdbf30ec 100644 --- a/src/server.c +++ b/src/server.c @@ -3976,7 +3976,7 @@ sds genRedisInfoString(const char *section) { "client_recent_max_output_buffer:%zu\r\n" "blocked_clients:%d\r\n" "tracking_clients:%d\r\n" - "clients_in_timeout_table:%ld\r\n", + "clients_in_timeout_table:%llu\r\n", listLength(server.clients)-listLength(server.slaves), maxin, maxout, server.blocked_clients, From a3decf12af51c12457e435826bdd65c8a1cbdd9a Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 31 Mar 2020 17:41:23 +0200 Subject: [PATCH 0228/1280] cast raxSize() to avoid warning with format spec. --- src/server.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/server.c b/src/server.c index 4fdbf30ec..c89e9c075 100644 --- a/src/server.c +++ b/src/server.c @@ -3981,7 +3981,7 @@ sds genRedisInfoString(const char *section) { maxin, maxout, server.blocked_clients, server.tracking_clients, - raxSize(server.clients_timeout_table)); + (unsigned long long) raxSize(server.clients_timeout_table)); } /* Memory */ From bc8c56a71a4bb6dded37da660d83a2e5f9617a54 Mon Sep 17 00:00:00 2001 From: Guy Benoish Date: Tue, 21 Jan 2020 15:09:42 +0530 Subject: [PATCH 0229/1280] Fix memory corruption in moduleHandleBlockedClients By using a "circular BRPOPLPUSH"-like scenario it was possible the get the same client on db->blocking_keys twice (See comment in moduleTryServeClientBlockedOnKey) The fix was actually already implememnted in moduleTryServeClientBlockedOnKey but it had a bug: the funxction should return 0 or 1 (not OK or ERR) Other changes: 1. Added two commands to blockonkeys.c test module (To reproduce the case described above) 2. Simplify blockonkeys.c in order to make testing easier 3. cast raxSize() to avoid warning with format spec --- src/module.c | 18 +++- src/server.c | 2 +- tests/modules/blockonkeys.c | 122 ++++++++++++++++++++++----- tests/unit/moduleapi/blockonkeys.tcl | 55 +++++++----- 4 files changed, 150 insertions(+), 47 deletions(-) diff --git a/src/module.c b/src/module.c index 61dc25169..85e1497fd 100644 --- a/src/module.c +++ b/src/module.c @@ -4393,14 +4393,26 @@ RedisModuleBlockedClient *moduleBlockClient(RedisModuleCtx *ctx, RedisModuleCmdF * can really be unblocked, since the module was able to serve the client. * If the callback returns REDISMODULE_OK, then the client can be unblocked, * otherwise the client remains blocked and we'll retry again when one of - * the keys it blocked for becomes "ready" again. */ + * the keys it blocked for becomes "ready" again. + * This function returns 1 if client was served (and should be unblocked) */ int moduleTryServeClientBlockedOnKey(client *c, robj *key) { int served = 0; RedisModuleBlockedClient *bc = c->bpop.module_blocked_handle; /* Protect against re-processing: don't serve clients that are already * in the unblocking list for any reason (including RM_UnblockClient() - * explicit call). */ - if (bc->unblocked) return REDISMODULE_ERR; + * explicit call). + * For example, the following pathological case: + * Assume a module called LIST implements the same command as + * the Redis list data type. + * LIST.BRPOPLPUSH src dst 0 ('src' goes into db->blocking_keys) + * LIST.BRPOPLPUSH dst src 0 ('dst' goes into db->blocking_keys) + * LIST.LPUSH src foo + * 'src' is in db->blocking_keys after the first BRPOPLPUSH is served + * (and stays there until the next beforeSleep). + * The second BRPOPLPUSH will signal 'src' as ready, leading to the + * unblocking of the already unblocked (and worst, freed) reply_client + * of the first BRPOPLPUSH. */ + if (bc->unblocked) return 0; RedisModuleCtx ctx = REDISMODULE_CTX_INIT; ctx.flags |= REDISMODULE_CTX_BLOCKED_REPLY; ctx.blocked_ready_key = key; diff --git a/src/server.c b/src/server.c index 4fdbf30ec..5d8b4fde9 100644 --- a/src/server.c +++ b/src/server.c @@ -3981,7 +3981,7 @@ sds genRedisInfoString(const char *section) { maxin, maxout, server.blocked_clients, server.tracking_clients, - raxSize(server.clients_timeout_table)); + (unsigned long long)raxSize(server.clients_timeout_table)); } /* Memory */ diff --git a/tests/modules/blockonkeys.c b/tests/modules/blockonkeys.c index 10dc65b1a..94f31d455 100644 --- a/tests/modules/blockonkeys.c +++ b/tests/modules/blockonkeys.c @@ -109,41 +109,33 @@ int fsl_push(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { return RedisModule_ReplyWithError(ctx,"ERR new element has to be greater than the head element"); fsl->list[fsl->length++] = ele; - - if (fsl->length >= 2) - RedisModule_SignalKeyAsReady(ctx, argv[1]); + RedisModule_SignalKeyAsReady(ctx, argv[1]); return RedisModule_ReplyWithSimpleString(ctx, "OK"); } -int bpop2_reply_callback(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { +int bpop_reply_callback(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { REDISMODULE_NOT_USED(argv); REDISMODULE_NOT_USED(argc); RedisModuleString *keyname = RedisModule_GetBlockedClientReadyKey(ctx); fsl_t *fsl; - if (!get_fsl(ctx, keyname, REDISMODULE_READ, 0, &fsl, 0)) + if (!get_fsl(ctx, keyname, REDISMODULE_READ, 0, &fsl, 0) || !fsl) return REDISMODULE_ERR; - if (!fsl || fsl->length < 2) - return REDISMODULE_ERR; - - RedisModule_ReplyWithArray(ctx, 2); - RedisModule_ReplyWithLongLong(ctx, fsl->list[--fsl->length]); RedisModule_ReplyWithLongLong(ctx, fsl->list[--fsl->length]); return REDISMODULE_OK; } -int bpop2_timeout_callback(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { +int bpop_timeout_callback(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { REDISMODULE_NOT_USED(argv); REDISMODULE_NOT_USED(argc); return RedisModule_ReplyWithSimpleString(ctx, "Request timedout"); } - -/* FSL.BPOP2 - Block clients until list has two or more elements. +/* FSL.BPOP - Block clients until list has two or more elements. * When that happens, unblock client and pop the last two elements (from the right). */ -int fsl_bpop2(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { +int fsl_bpop(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { if (argc != 3) return RedisModule_WrongArity(ctx); @@ -155,13 +147,10 @@ int fsl_bpop2(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { if (!get_fsl(ctx, argv[1], REDISMODULE_READ, 0, &fsl, 1)) return REDISMODULE_OK; - if (!fsl || fsl->length < 2) { - /* Key is empty or has <2 elements, we must block */ - RedisModule_BlockClientOnKeys(ctx, bpop2_reply_callback, bpop2_timeout_callback, + if (!fsl) { + RedisModule_BlockClientOnKeys(ctx, bpop_reply_callback, bpop_timeout_callback, NULL, timeout, &argv[1], 1, NULL); } else { - RedisModule_ReplyWithArray(ctx, 2); - RedisModule_ReplyWithLongLong(ctx, fsl->list[--fsl->length]); RedisModule_ReplyWithLongLong(ctx, fsl->list[--fsl->length]); } @@ -175,10 +164,10 @@ int bpopgt_reply_callback(RedisModuleCtx *ctx, RedisModuleString **argv, int arg long long *pgt = RedisModule_GetBlockedClientPrivateData(ctx); fsl_t *fsl; - if (!get_fsl(ctx, keyname, REDISMODULE_READ, 0, &fsl, 0)) + if (!get_fsl(ctx, keyname, REDISMODULE_READ, 0, &fsl, 0) || !fsl) return REDISMODULE_ERR; - if (!fsl || fsl->list[fsl->length-1] <= *pgt) + if (fsl->list[fsl->length-1] <= *pgt) return REDISMODULE_ERR; RedisModule_ReplyWithLongLong(ctx, fsl->list[--fsl->length]); @@ -218,7 +207,6 @@ int fsl_bpopgt(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { /* We use malloc so the tests in blockedonkeys.tcl can check for memory leaks */ long long *pgt = RedisModule_Alloc(sizeof(long long)); *pgt = gt; - /* Key is empty or has <2 elements, we must block */ RedisModule_BlockClientOnKeys(ctx, bpopgt_reply_callback, bpopgt_timeout_callback, bpopgt_free_privdata, timeout, &argv[1], 1, pgt); } else { @@ -228,6 +216,88 @@ int fsl_bpopgt(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { return REDISMODULE_OK; } +int bpoppush_reply_callback(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { + REDISMODULE_NOT_USED(argv); + REDISMODULE_NOT_USED(argc); + RedisModuleString *src_keyname = RedisModule_GetBlockedClientReadyKey(ctx); + RedisModuleString *dst_keyname = RedisModule_GetBlockedClientPrivateData(ctx); + + fsl_t *src; + if (!get_fsl(ctx, src_keyname, REDISMODULE_READ, 0, &src, 0) || !src) + return REDISMODULE_ERR; + + fsl_t *dst; + if (!get_fsl(ctx, dst_keyname, REDISMODULE_WRITE, 1, &dst, 0) || !dst) + return REDISMODULE_ERR; + + long long ele = src->list[--src->length]; + dst->list[dst->length++] = ele; + RedisModule_SignalKeyAsReady(ctx, dst_keyname); + return RedisModule_ReplyWithLongLong(ctx, ele); +} + +int bpoppush_timeout_callback(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { + REDISMODULE_NOT_USED(argv); + REDISMODULE_NOT_USED(argc); + return RedisModule_ReplyWithSimpleString(ctx, "Request timedout"); +} + +void bpoppush_free_privdata(RedisModuleCtx *ctx, void *privdata) { + RedisModule_FreeString(ctx, privdata); +} + +/* FSL.BPOPPUSH - Block clients until has an element. + * When that happens, unblock client, pop the last element from and push it to + * (from the right). */ +int fsl_bpoppush(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { + if (argc != 4) + return RedisModule_WrongArity(ctx); + + long long timeout; + if (RedisModule_StringToLongLong(argv[3],&timeout) != REDISMODULE_OK || timeout < 0) + return RedisModule_ReplyWithError(ctx,"ERR invalid timeout"); + + fsl_t *src; + if (!get_fsl(ctx, argv[1], REDISMODULE_READ, 0, &src, 1)) + return REDISMODULE_OK; + + if (!src) { + /* Retain string for reply callback */ + RedisModule_RetainString(ctx, argv[2]); + /* Key is empty, we must block */ + RedisModule_BlockClientOnKeys(ctx, bpoppush_reply_callback, bpoppush_timeout_callback, + bpoppush_free_privdata, timeout, &argv[1], 1, argv[2]); + } else { + fsl_t *dst; + if (!get_fsl(ctx, argv[2], REDISMODULE_WRITE, 1, &dst, 1)) + return REDISMODULE_OK; + long long ele = src->list[--src->length]; + dst->list[dst->length++] = ele; + RedisModule_SignalKeyAsReady(ctx, argv[2]); + RedisModule_ReplyWithLongLong(ctx, ele); + } + + return REDISMODULE_OK; +} + +/* FSL.GETALL - Reply with an array containing all elements. */ +int fsl_getall(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { + if (argc != 2) + return RedisModule_WrongArity(ctx); + + fsl_t *fsl; + if (!get_fsl(ctx, argv[1], REDISMODULE_READ, 0, &fsl, 1)) + return REDISMODULE_OK; + + if (!fsl) + return RedisModule_ReplyWithArray(ctx, 0); + + RedisModule_ReplyWithArray(ctx, fsl->length); + for (int i = 0; i < fsl->length; i++) + RedisModule_ReplyWithLongLong(ctx, fsl->list[i]); + return REDISMODULE_OK; +} + int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { REDISMODULE_NOT_USED(argv); REDISMODULE_NOT_USED(argc); @@ -252,11 +322,17 @@ int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) if (RedisModule_CreateCommand(ctx,"fsl.push",fsl_push,"",0,0,0) == REDISMODULE_ERR) return REDISMODULE_ERR; - if (RedisModule_CreateCommand(ctx,"fsl.bpop2",fsl_bpop2,"",0,0,0) == REDISMODULE_ERR) + if (RedisModule_CreateCommand(ctx,"fsl.bpop",fsl_bpop,"",0,0,0) == REDISMODULE_ERR) return REDISMODULE_ERR; if (RedisModule_CreateCommand(ctx,"fsl.bpopgt",fsl_bpopgt,"",0,0,0) == REDISMODULE_ERR) return REDISMODULE_ERR; + if (RedisModule_CreateCommand(ctx,"fsl.bpoppush",fsl_bpoppush,"",0,0,0) == REDISMODULE_ERR) + return REDISMODULE_ERR; + + if (RedisModule_CreateCommand(ctx,"fsl.getall",fsl_getall,"",0,0,0) == REDISMODULE_ERR) + return REDISMODULE_ERR; + return REDISMODULE_OK; } diff --git a/tests/unit/moduleapi/blockonkeys.tcl b/tests/unit/moduleapi/blockonkeys.tcl index b380227e0..c8b8f23ed 100644 --- a/tests/unit/moduleapi/blockonkeys.tcl +++ b/tests/unit/moduleapi/blockonkeys.tcl @@ -3,37 +3,53 @@ set testmodule [file normalize tests/modules/blockonkeys.so] start_server {tags {"modules"}} { r module load $testmodule + test "Module client blocked on keys: Circular BPOPPUSH" { + set rd1 [redis_deferring_client] + set rd2 [redis_deferring_client] + + r del src dst + + $rd1 fsl.bpoppush src dst 0 + $rd2 fsl.bpoppush dst src 0 + + r fsl.push src 42 + + assert_equal {42} [r fsl.getall src] + assert_equal {} [r fsl.getall dst] + } + + test "Module client blocked on keys: Self-referential BPOPPUSH" { + set rd1 [redis_deferring_client] + + r del src + + $rd1 fsl.bpoppush src src 0 + + r fsl.push src 42 + + assert_equal {42} [r fsl.getall src] + } + test {Module client blocked on keys (no metadata): No block} { r del k r fsl.push k 33 r fsl.push k 34 - r fsl.bpop2 k 0 - } {34 33} + r fsl.bpop k 0 + } {34} test {Module client blocked on keys (no metadata): Timeout} { r del k set rd [redis_deferring_client] - r fsl.push k 33 - $rd fsl.bpop2 k 1 + $rd fsl.bpop k 1 assert_equal {Request timedout} [$rd read] } - test {Module client blocked on keys (no metadata): Blocked, case 1} { + test {Module client blocked on keys (no metadata): Blocked} { r del k set rd [redis_deferring_client] - r fsl.push k 33 - $rd fsl.bpop2 k 0 + $rd fsl.bpop k 0 r fsl.push k 34 - assert_equal {34 33} [$rd read] - } - - test {Module client blocked on keys (no metadata): Blocked, case 2} { - r del k - set rd [redis_deferring_client] - r fsl.push k 33 - r fsl.push k 34 - $rd fsl.bpop2 k 0 - assert_equal {34 33} [$rd read] + assert_equal {34} [$rd read] } test {Module client blocked on keys (with metadata): No block} { @@ -108,13 +124,12 @@ start_server {tags {"modules"}} { test {Module client blocked on keys does not wake up on wrong type} { r del k set rd [redis_deferring_client] - $rd fsl.bpop2 k 0 + $rd fsl.bpop k 0 r lpush k 12 r lpush k 13 r lpush k 14 r del k - r fsl.push k 33 r fsl.push k 34 - assert_equal {34 33} [$rd read] + assert_equal {34} [$rd read] } } From d77fd23ae2a1c5c1d441f1b8f41748e395a3775f Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 1 Apr 2020 16:10:18 +0200 Subject: [PATCH 0230/1280] LCS: initial functionality implemented. --- src/db.c | 26 +++++++++++ src/server.c | 6 ++- src/server.h | 2 + src/t_string.c | 123 +++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 156 insertions(+), 1 deletion(-) diff --git a/src/db.c b/src/db.c index 211bb978d..bfa39564e 100644 --- a/src/db.c +++ b/src/db.c @@ -1554,6 +1554,32 @@ int *georadiusGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numk return keys; } +/* LCS ... [STOREIDX ] ... */ +int *lcsGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numkeys) +{ + int i; + int *keys; + UNUSED(cmd); + + /* We need to parse the options of the command in order to check for the + * "STOREIDX" argument before the STRINGS argument. */ + for (i = 1; i < argc; i++) { + char *arg = argv[i]->ptr; + int moreargs = (argc-1) - i; + + if (!strcasecmp(arg, "strings")) { + break; + } else if (!strcasecmp(arg, "storeidx") && moreargs) { + keys = getKeysTempBuffer; + keys[0] = i+1; + *numkeys = 1; + return keys; + } + } + *numkeys = 0; + return NULL; +} + /* Helper function to extract keys from memory command. * MEMORY USAGE */ int *memoryGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numkeys) { diff --git a/src/server.c b/src/server.c index c89e9c075..d06a9e29a 100644 --- a/src/server.c +++ b/src/server.c @@ -1004,7 +1004,11 @@ struct redisCommand redisCommandTable[] = { {"acl",aclCommand,-2, "admin no-script no-slowlog ok-loading ok-stale", - 0,NULL,0,0,0,0,0,0} + 0,NULL,0,0,0,0,0,0}, + + {"lcs",lcsCommand,-4, + "write use-memory @string", + 0,lcsGetKeys,0,0,0,0,0,0} }; /*============================ Utility functions ============================ */ diff --git a/src/server.h b/src/server.h index c4db4278e..1472bcee7 100644 --- a/src/server.h +++ b/src/server.h @@ -2101,6 +2101,7 @@ int *migrateGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numkey int *georadiusGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numkeys); int *xreadGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numkeys); int *memoryGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numkeys); +int *lcsGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numkeys); /* Cluster */ void clusterInit(void); @@ -2372,6 +2373,7 @@ void xdelCommand(client *c); void xtrimCommand(client *c); void lolwutCommand(client *c); void aclCommand(client *c); +void lcsCommand(client *c); #if defined(__GNUC__) void *calloc(size_t count, size_t size) __attribute__ ((deprecated)); diff --git a/src/t_string.c b/src/t_string.c index 8ccd69eb9..e19647845 100644 --- a/src/t_string.c +++ b/src/t_string.c @@ -479,3 +479,126 @@ void strlenCommand(client *c) { checkType(c,o,OBJ_STRING)) return; addReplyLongLong(c,stringObjectLen(o)); } + +/* LCS -- Longest common subsequence. + * + * LCS [LEN] [IDX] [STOREIDX ] STRINGS */ +void lcsCommand(client *c) { + uint32_t i, j; + sds a = NULL, b = NULL; + int getlen = 0, getidx = 0; + robj *idxkey = NULL; /* STOREIDX will set this and getidx to 1. */ + + for (j = 1; j < (uint32_t)c->argc; j++) { + char *opt = c->argv[j]->ptr; + int moreargs = (c->argc-1) - j; + + if (!strcasecmp(opt,"IDX")) { + getidx = 1; + } else if (!strcasecmp(opt,"STOREIDX") && moreargs) { + getidx = 1; + idxkey = c->argv[j+1]; + j++; + } else if (!strcasecmp(opt,"LEN")) { + getlen++; + } else if (!strcasecmp(opt,"STRINGS")) { + if (moreargs != 2) { + addReplyError(c,"LCS requires exactly two strings"); + return; + } + a = c->argv[j+1]->ptr; + b = c->argv[j+2]->ptr; + j += 2; + } else { + addReply(c,shared.syntaxerr); + return; + } + } + + /* Complain if the user didn't pass the STRING option. */ + if (a == NULL) { + addReplyError(c,"STRINGS is mandatory"); + return; + } + + /* Compute the LCS using the vanilla dynamic programming technique of + * building a table of LCS(x,y) substrings. */ + uint32_t alen = sdslen(a); + uint32_t blen = sdslen(b); + + /* Setup an uint32_t array to store at LCS[i,j] the length of the + * LCS A0..i-1, B0..j-1. Note that we have a linear array here, so + * we index it as LCS[i+alen*j] */ + uint32_t *lcs = zmalloc((alen+1)*(blen+1)*sizeof(uint32_t)); + #define LCS(A,B) lcs[(A)+((B)*(alen+1))] + + /* Start building the LCS table. */ + for (uint32_t i = 0; i <= alen; i++) { + for (uint32_t j = 0; j <= blen; j++) { + if (i == 0 || j == 0) { + /* If one substring has length of zero, the + * LCS length is zero. */ + LCS(i,j) = 0; + } else if (a[i-1] == b[j-1]) { + /* The len LCS (and the LCS itself) of two + * sequences with the same final character, is the + * LCS of the two sequences without the last char + * plus that last char. */ + LCS(i,j) = LCS(i-1,j-1)+1; + } else { + /* If the last character is different, take the longest + * between the LCS of the first string and the second + * minus the last char, and the reverse. */ + uint32_t lcs1 = LCS(i-1,j); + uint32_t lcs2 = LCS(i,j-1); + LCS(i,j) = lcs1 > lcs2 ? lcs1 : lcs2; + } + } + } + + /* Store the actual LCS string in "result" if needed. We create + * it backward, but the length is already known, we store it into idx. */ + uint32_t idx = LCS(alen,blen); + sds result = NULL; + + /* Do we need to compute the actual LCS string? Allocate it in that case. */ + int computelcs = getidx || !getlen; + if (computelcs) result = sdsnewlen(SDS_NOINIT,idx); + + i = alen, j = blen; + while (computelcs && i > 0 && j > 0) { + if (a[i-1] == b[j-1]) { + /* If there is a match, store the character and reduce + * the indexes to look for a new match. */ + result[idx-1] = a[i-1]; + idx--; + i--; + j--; + } else { + /* Otherwise reduce i and j depending on the largest + * LCS between, to understand what direction we need to go. */ + uint32_t lcs1 = LCS(i-1,j); + uint32_t lcs2 = LCS(i,j-1); + if (lcs1 > lcs2) + i--; + else + j--; + } + } + + /* Signal modified key, increment dirty, ... */ + + /* Reply depending on the given options. */ + if (getlen) { + addReplyLongLong(c,LCS(alen,blen)); + } else { + addReplyBulkSds(c,result); + result = NULL; + } + + /* Cleanup. */ + sdsfree(result); + zfree(lcs); + return; +} + From b5ae25332fc38c49270269152cc5b7ace2626bb8 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 1 Apr 2020 17:11:31 +0200 Subject: [PATCH 0231/1280] LCS: implement range indexes option. --- src/t_string.c | 68 +++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 59 insertions(+), 9 deletions(-) diff --git a/src/t_string.c b/src/t_string.c index e19647845..6c096a539 100644 --- a/src/t_string.c +++ b/src/t_string.c @@ -482,7 +482,7 @@ void strlenCommand(client *c) { /* LCS -- Longest common subsequence. * - * LCS [LEN] [IDX] [STOREIDX ] STRINGS */ + * LCS [IDX] [STOREIDX ] STRINGS */ void lcsCommand(client *c) { uint32_t i, j; sds a = NULL, b = NULL; @@ -495,12 +495,12 @@ void lcsCommand(client *c) { if (!strcasecmp(opt,"IDX")) { getidx = 1; + } else if (!strcasecmp(opt,"LEN")) { + getlen = 1; } else if (!strcasecmp(opt,"STOREIDX") && moreargs) { getidx = 1; idxkey = c->argv[j+1]; j++; - } else if (!strcasecmp(opt,"LEN")) { - getlen++; } else if (!strcasecmp(opt,"STRINGS")) { if (moreargs != 2) { addReplyError(c,"LCS requires exactly two strings"); @@ -515,10 +515,17 @@ void lcsCommand(client *c) { } } - /* Complain if the user didn't pass the STRING option. */ + /* Complain if the user passed ambiguous parameters. */ if (a == NULL) { addReplyError(c,"STRINGS is mandatory"); return; + } else if (getlen && (getidx && idxkey == NULL)) { + addReplyError(c, + "If you want both the LEN and indexes, please " + "store the indexes into a key with STOREIDX. However note " + "that the IDX output also includes both the LCS string and " + "its length"); + return; } /* Compute the LCS using the vanilla dynamic programming technique of @@ -559,21 +566,62 @@ void lcsCommand(client *c) { /* Store the actual LCS string in "result" if needed. We create * it backward, but the length is already known, we store it into idx. */ uint32_t idx = LCS(alen,blen); - sds result = NULL; + sds result = NULL; /* Resulting LCS string. */ + void *arraylenptr = NULL; /* Deffered length of the array for IDX. */ + uint32_t arange_start = alen, /* alen signals that values are not set. */ + arange_end = 0, + brange_start = 0, + brange_end = 0; /* Do we need to compute the actual LCS string? Allocate it in that case. */ int computelcs = getidx || !getlen; if (computelcs) result = sdsnewlen(SDS_NOINIT,idx); + /* Start with a deferred array if we have to emit the ranges. */ + uint32_t arraylen = 0; /* Number of ranges emitted in the array. */ + if (getidx && idxkey == NULL) + arraylenptr = addReplyDeferredLen(c); + i = alen, j = blen; while (computelcs && i > 0 && j > 0) { if (a[i-1] == b[j-1]) { /* If there is a match, store the character and reduce * the indexes to look for a new match. */ result[idx-1] = a[i-1]; - idx--; - i--; - j--; + /* Track the current range. */ + int emit_range = 0; + if (arange_start == alen) { + arange_start = i-1; + arange_end = i-1; + brange_start = j-1; + brange_end = j-1; + if (i == 0 || j == 0) emit_range = 1; + } else { + /* Let's see if we can extend the range backward since + * it is contiguous. */ + if (arange_start == i && brange_start == j) { + arange_start--; + brange_start--; + } else { + emit_range = 1; + } + } + + /* Emit the current range if needed. */ + if (emit_range) { + if (arraylenptr) { + addReplyArrayLen(c,2); + addReplyArrayLen(c,2); + addReplyLongLong(c,arange_start); + addReplyLongLong(c,arange_end); + addReplyArrayLen(c,2); + addReplyLongLong(c,brange_start); + addReplyLongLong(c,brange_end); + } + arange_start = alen; /* Restart at the next match. */ + arraylen++; + } + idx--; i--; j--; } else { /* Otherwise reduce i and j depending on the largest * LCS between, to understand what direction we need to go. */ @@ -589,7 +637,9 @@ void lcsCommand(client *c) { /* Signal modified key, increment dirty, ... */ /* Reply depending on the given options. */ - if (getlen) { + if (arraylenptr) { + setDeferredArrayLen(c,arraylenptr,arraylen); + } else if (getlen) { addReplyLongLong(c,LCS(alen,blen)); } else { addReplyBulkSds(c,result); From 223c9cb6494d7ca5d03622eda588ebc9041dec23 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 1 Apr 2020 17:14:36 +0200 Subject: [PATCH 0232/1280] LCS: fix emission of last range starting at index 0. --- src/t_string.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/t_string.c b/src/t_string.c index 6c096a539..caccd11f1 100644 --- a/src/t_string.c +++ b/src/t_string.c @@ -595,7 +595,6 @@ void lcsCommand(client *c) { arange_end = i-1; brange_start = j-1; brange_end = j-1; - if (i == 0 || j == 0) emit_range = 1; } else { /* Let's see if we can extend the range backward since * it is contiguous. */ @@ -606,6 +605,7 @@ void lcsCommand(client *c) { emit_range = 1; } } + if (arange_start == 0 || brange_start == 0) emit_range = 1; /* Emit the current range if needed. */ if (emit_range) { From e8d4fd71f8c1720b101f0aad36fcdc775eef96ac Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 1 Apr 2020 17:36:32 +0200 Subject: [PATCH 0233/1280] LCS: other fixes to range emission. --- src/t_string.c | 36 ++++++++++++++++++++---------------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/src/t_string.c b/src/t_string.c index caccd11f1..30999ee70 100644 --- a/src/t_string.c +++ b/src/t_string.c @@ -584,12 +584,13 @@ void lcsCommand(client *c) { i = alen, j = blen; while (computelcs && i > 0 && j > 0) { + int emit_range = 0; if (a[i-1] == b[j-1]) { /* If there is a match, store the character and reduce * the indexes to look for a new match. */ result[idx-1] = a[i-1]; + /* Track the current range. */ - int emit_range = 0; if (arange_start == alen) { arange_start = i-1; arange_end = i-1; @@ -605,22 +606,9 @@ void lcsCommand(client *c) { emit_range = 1; } } + /* Emit the range if we matched with the first byte of + * one of the two strings. We'll exit the loop ASAP. */ if (arange_start == 0 || brange_start == 0) emit_range = 1; - - /* Emit the current range if needed. */ - if (emit_range) { - if (arraylenptr) { - addReplyArrayLen(c,2); - addReplyArrayLen(c,2); - addReplyLongLong(c,arange_start); - addReplyLongLong(c,arange_end); - addReplyArrayLen(c,2); - addReplyLongLong(c,brange_start); - addReplyLongLong(c,brange_end); - } - arange_start = alen; /* Restart at the next match. */ - arraylen++; - } idx--; i--; j--; } else { /* Otherwise reduce i and j depending on the largest @@ -631,6 +619,22 @@ void lcsCommand(client *c) { i--; else j--; + if (arange_start != alen) emit_range = 1; + } + + /* Emit the current range if needed. */ + if (emit_range) { + if (arraylenptr) { + addReplyArrayLen(c,2); + addReplyArrayLen(c,2); + addReplyLongLong(c,arange_start); + addReplyLongLong(c,arange_end); + addReplyArrayLen(c,2); + addReplyLongLong(c,brange_start); + addReplyLongLong(c,brange_end); + } + arange_start = alen; /* Restart at the next match. */ + arraylen++; } } From f882f5bcc4c1c56c25a84c538accdaebb7b74b10 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 1 Apr 2020 22:11:59 +0200 Subject: [PATCH 0234/1280] LCS: implement KEYS option. --- src/t_string.c | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/src/t_string.c b/src/t_string.c index 30999ee70..0b159e6b3 100644 --- a/src/t_string.c +++ b/src/t_string.c @@ -482,12 +482,13 @@ void strlenCommand(client *c) { /* LCS -- Longest common subsequence. * - * LCS [IDX] [STOREIDX ] STRINGS */ + * LCS [IDX] [STOREIDX ] STRINGS | KEYS */ void lcsCommand(client *c) { uint32_t i, j; sds a = NULL, b = NULL; int getlen = 0, getidx = 0; robj *idxkey = NULL; /* STOREIDX will set this and getidx to 1. */ + robj *obja = NULL, *objb = NULL; for (j = 1; j < (uint32_t)c->argc; j++) { char *opt = c->argv[j]->ptr; @@ -509,6 +510,18 @@ void lcsCommand(client *c) { a = c->argv[j+1]->ptr; b = c->argv[j+2]->ptr; j += 2; + } else if (!strcasecmp(opt,"KEYS")) { + if (moreargs != 2) { + addReplyError(c,"LCS requires exactly two keys"); + return; + } + obja = lookupKeyRead(c->db,c->argv[j+1]); + objb = lookupKeyRead(c->db,c->argv[j+2]); + obja = obja ? getDecodedObject(obja) : createStringObject("",0); + objb = objb ? getDecodedObject(objb) : createStringObject("",0); + a = obja->ptr; + b = objb->ptr; + j += 2; } else { addReply(c,shared.syntaxerr); return; @@ -517,7 +530,8 @@ void lcsCommand(client *c) { /* Complain if the user passed ambiguous parameters. */ if (a == NULL) { - addReplyError(c,"STRINGS is mandatory"); + addReplyError(c,"Please specify two strings: " + "STRINGS or KEYS options are mandatory"); return; } else if (getlen && (getidx && idxkey == NULL)) { addReplyError(c, @@ -651,6 +665,8 @@ void lcsCommand(client *c) { } /* Cleanup. */ + if (obja) decrRefCount(obja); + if (objb) decrRefCount(objb); sdsfree(result); zfree(lcs); return; From 8699d75e29555cd1aa3f1503504da5a40de3ce2b Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 1 Apr 2020 23:45:07 +0200 Subject: [PATCH 0235/1280] LCS: 7x speedup by accessing the array with better locality. --- src/t_string.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/t_string.c b/src/t_string.c index 0b159e6b3..b715e144a 100644 --- a/src/t_string.c +++ b/src/t_string.c @@ -551,7 +551,7 @@ void lcsCommand(client *c) { * LCS A0..i-1, B0..j-1. Note that we have a linear array here, so * we index it as LCS[i+alen*j] */ uint32_t *lcs = zmalloc((alen+1)*(blen+1)*sizeof(uint32_t)); - #define LCS(A,B) lcs[(A)+((B)*(alen+1))] + #define LCS(A,B) lcs[(B)+((A)*(blen+1))] /* Start building the LCS table. */ for (uint32_t i = 0; i <= alen; i++) { From fba7d2be4d4826a1348dafc872820463d85e5e2e Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 2 Apr 2020 11:20:09 +0200 Subject: [PATCH 0236/1280] Simplify comment in moduleTryServeClientBlockedOnKey(). --- src/module.c | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/src/module.c b/src/module.c index 85e1497fd..16d24152e 100644 --- a/src/module.c +++ b/src/module.c @@ -4398,21 +4398,12 @@ RedisModuleBlockedClient *moduleBlockClient(RedisModuleCtx *ctx, RedisModuleCmdF int moduleTryServeClientBlockedOnKey(client *c, robj *key) { int served = 0; RedisModuleBlockedClient *bc = c->bpop.module_blocked_handle; + /* Protect against re-processing: don't serve clients that are already * in the unblocking list for any reason (including RM_UnblockClient() - * explicit call). - * For example, the following pathological case: - * Assume a module called LIST implements the same command as - * the Redis list data type. - * LIST.BRPOPLPUSH src dst 0 ('src' goes into db->blocking_keys) - * LIST.BRPOPLPUSH dst src 0 ('dst' goes into db->blocking_keys) - * LIST.LPUSH src foo - * 'src' is in db->blocking_keys after the first BRPOPLPUSH is served - * (and stays there until the next beforeSleep). - * The second BRPOPLPUSH will signal 'src' as ready, leading to the - * unblocking of the already unblocked (and worst, freed) reply_client - * of the first BRPOPLPUSH. */ + * explicit call). See #6798. */ if (bc->unblocked) return 0; + RedisModuleCtx ctx = REDISMODULE_CTX_INIT; ctx.flags |= REDISMODULE_CTX_BLOCKED_REPLY; ctx.blocked_ready_key = key; From a9954197aa545c4f676e647de509124052e602c3 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 2 Apr 2020 13:37:35 +0200 Subject: [PATCH 0237/1280] LCS: MINMATCHLEN and WITHMATCHLEN options. --- src/t_string.c | 35 ++++++++++++++++++++++++----------- 1 file changed, 24 insertions(+), 11 deletions(-) diff --git a/src/t_string.c b/src/t_string.c index b715e144a..6b30c9751 100644 --- a/src/t_string.c +++ b/src/t_string.c @@ -482,11 +482,13 @@ void strlenCommand(client *c) { /* LCS -- Longest common subsequence. * - * LCS [IDX] [STOREIDX ] STRINGS | KEYS */ + * LCS [IDX] [STOREIDX ] [MINMATCHLEN ] + * STRINGS | KEYS */ void lcsCommand(client *c) { uint32_t i, j; + long long minmatchlen = 0; sds a = NULL, b = NULL; - int getlen = 0, getidx = 0; + int getlen = 0, getidx = 0, withmatchlen = 0; robj *idxkey = NULL; /* STOREIDX will set this and getidx to 1. */ robj *obja = NULL, *objb = NULL; @@ -498,10 +500,17 @@ void lcsCommand(client *c) { getidx = 1; } else if (!strcasecmp(opt,"LEN")) { getlen = 1; + } else if (!strcasecmp(opt,"WITHMATCHLEN")) { + withmatchlen = 1; } else if (!strcasecmp(opt,"STOREIDX") && moreargs) { getidx = 1; idxkey = c->argv[j+1]; j++; + } else if (!strcasecmp(opt,"MINMATCHLEN") && moreargs) { + if (getLongLongFromObjectOrReply(c,c->argv[j+1],&minmatchlen,NULL) + != C_OK) return; + if (minmatchlen < 0) minmatchlen = 0; + j++; } else if (!strcasecmp(opt,"STRINGS")) { if (moreargs != 2) { addReplyError(c,"LCS requires exactly two strings"); @@ -637,18 +646,22 @@ void lcsCommand(client *c) { } /* Emit the current range if needed. */ + uint32_t match_len = arange_end - arange_start + 1; if (emit_range) { - if (arraylenptr) { - addReplyArrayLen(c,2); - addReplyArrayLen(c,2); - addReplyLongLong(c,arange_start); - addReplyLongLong(c,arange_end); - addReplyArrayLen(c,2); - addReplyLongLong(c,brange_start); - addReplyLongLong(c,brange_end); + if (minmatchlen == 0 || match_len >= minmatchlen) { + if (arraylenptr) { + addReplyArrayLen(c,2+withmatchlen); + addReplyArrayLen(c,2); + addReplyLongLong(c,arange_start); + addReplyLongLong(c,arange_end); + addReplyArrayLen(c,2); + addReplyLongLong(c,brange_start); + addReplyLongLong(c,brange_end); + if (withmatchlen) addReplyLongLong(c,match_len); + arraylen++; + } } arange_start = alen; /* Restart at the next match. */ - arraylen++; } } From 424facd7a1dc08174634e9ce520675db2617dbf6 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 2 Apr 2020 16:15:17 +0200 Subject: [PATCH 0238/1280] LCS: output LCS len as well in IDX mode. --- src/t_string.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/t_string.c b/src/t_string.c index 6b30c9751..fb53f7c54 100644 --- a/src/t_string.c +++ b/src/t_string.c @@ -602,8 +602,11 @@ void lcsCommand(client *c) { /* Start with a deferred array if we have to emit the ranges. */ uint32_t arraylen = 0; /* Number of ranges emitted in the array. */ - if (getidx && idxkey == NULL) + if (getidx && idxkey == NULL) { + addReplyMapLen(c,2); + addReplyBulkCString(c,"matches"); arraylenptr = addReplyDeferredLen(c); + } i = alen, j = blen; while (computelcs && i > 0 && j > 0) { @@ -669,6 +672,8 @@ void lcsCommand(client *c) { /* Reply depending on the given options. */ if (arraylenptr) { + addReplyBulkCString(c,"len"); + addReplyLongLong(c,LCS(alen,blen)); setDeferredArrayLen(c,arraylenptr,arraylen); } else if (getlen) { addReplyLongLong(c,LCS(alen,blen)); From 7894a37841b64ff5b16cb71477507edaaaa4cabb Mon Sep 17 00:00:00 2001 From: Guy Benoish Date: Tue, 25 Feb 2020 16:51:35 +0530 Subject: [PATCH 0239/1280] Stale replica should allow MULTI/EXEC Example: Client uses a pipe to send the following to a stale replica: MULTI .. do something ... DISCARD The replica will reply the MUTLI with -MASTERDOWN and execute the rest of the commands... A client using a pipe might not be aware that MULTI failed until it's too late. I can't think of a reason why MULTI/EXEC/DISCARD should not be executed on stale replicas... Also, enable MULTI/EXEC/DISCARD during loading --- src/server.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/server.c b/src/server.c index c89e9c075..ff7a39df5 100644 --- a/src/server.c +++ b/src/server.c @@ -672,15 +672,15 @@ struct redisCommand redisCommandTable[] = { 0,NULL,1,1,1,0,0,0}, {"multi",multiCommand,1, - "no-script fast @transaction", + "no-script fast ok-loading ok-stale @transaction", 0,NULL,0,0,0,0,0,0}, {"exec",execCommand,1, - "no-script no-monitor no-slowlog @transaction", + "no-script no-monitor no-slowlog ok-loading ok-stale @transaction", 0,NULL,0,0,0,0,0,0}, {"discard",discardCommand,1, - "no-script fast @transaction", + "no-script fast ok-loading ok-stale @transaction", 0,NULL,0,0,0,0,0,0}, {"sync",syncCommand,1, From 74daccece3410cffba16a9c5024c5b9042eb29a5 Mon Sep 17 00:00:00 2001 From: Guy Benoish Date: Thu, 2 Apr 2020 18:41:29 +0300 Subject: [PATCH 0240/1280] Fix no-negative-zero test --- tests/unit/type/incr.tcl | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/unit/type/incr.tcl b/tests/unit/type/incr.tcl index 63bf2e116..b7a135203 100644 --- a/tests/unit/type/incr.tcl +++ b/tests/unit/type/incr.tcl @@ -153,6 +153,7 @@ start_server {tags {"incr"}} { } {ERR*valid*} test {No negative zero} { + r del foo r incrbyfloat foo [expr double(1)/41] r incrbyfloat foo [expr double(-1)/41] r get foo From f2fd46e5d3fac63864b9ce598825204ed60b303f Mon Sep 17 00:00:00 2001 From: Xudong Zhang Date: Thu, 2 Apr 2020 23:43:19 +0800 Subject: [PATCH 0241/1280] fix integer overflow --- src/quicklist.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/quicklist.c b/src/quicklist.c index ae183ffd8..52e3988f5 100644 --- a/src/quicklist.c +++ b/src/quicklist.c @@ -110,7 +110,7 @@ quicklist *quicklistCreate(void) { return quicklist; } -#define COMPRESS_MAX (1 << QL_COMP_BITS) +#define COMPRESS_MAX ((1 << QL_COMP_BITS)-1) void quicklistSetCompressDepth(quicklist *quicklist, int compress) { if (compress > COMPRESS_MAX) { compress = COMPRESS_MAX; @@ -120,7 +120,7 @@ void quicklistSetCompressDepth(quicklist *quicklist, int compress) { quicklist->compress = compress; } -#define FILL_MAX (1 << (QL_FILL_BITS-1)) +#define FILL_MAX ((1 << (QL_FILL_BITS-1))-1) void quicklistSetFill(quicklist *quicklist, int fill) { if (fill > FILL_MAX) { fill = FILL_MAX; From 689c3f7678262ff75aa5c2299ff3813516318564 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 2 Apr 2020 21:17:31 +0200 Subject: [PATCH 0242/1280] LCS: fix stale comment. --- src/t_string.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/t_string.c b/src/t_string.c index fb53f7c54..80cdbb5e1 100644 --- a/src/t_string.c +++ b/src/t_string.c @@ -558,7 +558,7 @@ void lcsCommand(client *c) { /* Setup an uint32_t array to store at LCS[i,j] the length of the * LCS A0..i-1, B0..j-1. Note that we have a linear array here, so - * we index it as LCS[i+alen*j] */ + * we index it as LCS[j+(blen+1)*j] */ uint32_t *lcs = zmalloc((alen+1)*(blen+1)*sizeof(uint32_t)); #define LCS(A,B) lcs[(B)+((A)*(blen+1))] From 032cf29e8794e0a681b7b0967196e772a6fc1e2d Mon Sep 17 00:00:00 2001 From: Guy Benoish Date: Thu, 6 Feb 2020 15:13:52 +0530 Subject: [PATCH 0243/1280] Modules: Perform printf-like format checks in variadic API --- src/redismodule.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/redismodule.h b/src/redismodule.h index d26c41456..754ee4fc6 100644 --- a/src/redismodule.h +++ b/src/redismodule.h @@ -470,7 +470,7 @@ RedisModuleString *REDISMODULE_API_FUNC(RedisModule_CreateStringFromLongLong)(Re RedisModuleString *REDISMODULE_API_FUNC(RedisModule_CreateStringFromDouble)(RedisModuleCtx *ctx, double d); RedisModuleString *REDISMODULE_API_FUNC(RedisModule_CreateStringFromLongDouble)(RedisModuleCtx *ctx, long double ld, int humanfriendly); RedisModuleString *REDISMODULE_API_FUNC(RedisModule_CreateStringFromString)(RedisModuleCtx *ctx, const RedisModuleString *str); -RedisModuleString *REDISMODULE_API_FUNC(RedisModule_CreateStringPrintf)(RedisModuleCtx *ctx, const char *fmt, ...); +RedisModuleString *REDISMODULE_API_FUNC(RedisModule_CreateStringPrintf)(RedisModuleCtx *ctx, const char *fmt, ...); __attribute__ ((format (printf, 2, 3))); void REDISMODULE_API_FUNC(RedisModule_FreeString)(RedisModuleCtx *ctx, RedisModuleString *str); const char *REDISMODULE_API_FUNC(RedisModule_StringPtrLen)(const RedisModuleString *str, size_t *len); int REDISMODULE_API_FUNC(RedisModule_ReplyWithError)(RedisModuleCtx *ctx, const char *err); @@ -554,8 +554,8 @@ void REDISMODULE_API_FUNC(RedisModule_SaveLongDouble)(RedisModuleIO *io, long do long double REDISMODULE_API_FUNC(RedisModule_LoadLongDouble)(RedisModuleIO *io); void *REDISMODULE_API_FUNC(RedisModule_LoadDataTypeFromString)(const RedisModuleString *str, const RedisModuleType *mt); RedisModuleString *REDISMODULE_API_FUNC(RedisModule_SaveDataTypeToString)(RedisModuleCtx *ctx, void *data, const RedisModuleType *mt); -void REDISMODULE_API_FUNC(RedisModule_Log)(RedisModuleCtx *ctx, const char *level, const char *fmt, ...); -void REDISMODULE_API_FUNC(RedisModule_LogIOError)(RedisModuleIO *io, const char *levelstr, const char *fmt, ...); +void REDISMODULE_API_FUNC(RedisModule_Log)(RedisModuleCtx *ctx, const char *level, const char *fmt, ...); __attribute__ ((format (printf, 3, 4))); +void REDISMODULE_API_FUNC(RedisModule_LogIOError)(RedisModuleIO *io, const char *levelstr, const char *fmt, ...); __attribute__ ((format (printf, 3, 4))); void REDISMODULE_API_FUNC(RedisModule__Assert)(const char *estr, const char *file, int line); void REDISMODULE_API_FUNC(RedisModule_LatencyAddSample)(const char *event, mstime_t latency); int REDISMODULE_API_FUNC(RedisModule_StringAppendBuffer)(RedisModuleCtx *ctx, RedisModuleString *str, const char *buf, size_t len); From d03bc20d250395f369878fd46cd1d707c5cff0ac Mon Sep 17 00:00:00 2001 From: Guy Benoish Date: Fri, 3 Apr 2020 14:49:40 +0300 Subject: [PATCH 0244/1280] Use __attribute__ only if __GNUC__ is defined --- src/redismodule.h | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/src/redismodule.h b/src/redismodule.h index 754ee4fc6..23b4d26e0 100644 --- a/src/redismodule.h +++ b/src/redismodule.h @@ -470,7 +470,11 @@ RedisModuleString *REDISMODULE_API_FUNC(RedisModule_CreateStringFromLongLong)(Re RedisModuleString *REDISMODULE_API_FUNC(RedisModule_CreateStringFromDouble)(RedisModuleCtx *ctx, double d); RedisModuleString *REDISMODULE_API_FUNC(RedisModule_CreateStringFromLongDouble)(RedisModuleCtx *ctx, long double ld, int humanfriendly); RedisModuleString *REDISMODULE_API_FUNC(RedisModule_CreateStringFromString)(RedisModuleCtx *ctx, const RedisModuleString *str); -RedisModuleString *REDISMODULE_API_FUNC(RedisModule_CreateStringPrintf)(RedisModuleCtx *ctx, const char *fmt, ...); __attribute__ ((format (printf, 2, 3))); +#ifdef __GNUC__ +RedisModuleString *REDISMODULE_API_FUNC(RedisModule_CreateStringPrintf)(RedisModuleCtx *ctx, const char *fmt, ...) __attribute__ ((format (printf, 2, 3))); +#else +RedisModuleString *REDISMODULE_API_FUNC(RedisModule_CreateStringPrintf)(RedisModuleCtx *ctx, const char *fmt, ...); +#endif void REDISMODULE_API_FUNC(RedisModule_FreeString)(RedisModuleCtx *ctx, RedisModuleString *str); const char *REDISMODULE_API_FUNC(RedisModule_StringPtrLen)(const RedisModuleString *str, size_t *len); int REDISMODULE_API_FUNC(RedisModule_ReplyWithError)(RedisModuleCtx *ctx, const char *err); @@ -554,8 +558,13 @@ void REDISMODULE_API_FUNC(RedisModule_SaveLongDouble)(RedisModuleIO *io, long do long double REDISMODULE_API_FUNC(RedisModule_LoadLongDouble)(RedisModuleIO *io); void *REDISMODULE_API_FUNC(RedisModule_LoadDataTypeFromString)(const RedisModuleString *str, const RedisModuleType *mt); RedisModuleString *REDISMODULE_API_FUNC(RedisModule_SaveDataTypeToString)(RedisModuleCtx *ctx, void *data, const RedisModuleType *mt); -void REDISMODULE_API_FUNC(RedisModule_Log)(RedisModuleCtx *ctx, const char *level, const char *fmt, ...); __attribute__ ((format (printf, 3, 4))); -void REDISMODULE_API_FUNC(RedisModule_LogIOError)(RedisModuleIO *io, const char *levelstr, const char *fmt, ...); __attribute__ ((format (printf, 3, 4))); +#ifdef __GNUC__ +void REDISMODULE_API_FUNC(RedisModule_Log)(RedisModuleCtx *ctx, const char *level, const char *fmt, ...) __attribute__ ((format (printf, 3, 4))); +void REDISMODULE_API_FUNC(RedisModule_LogIOError)(RedisModuleIO *io, const char *levelstr, const char *fmt, ...) __attribute__ ((format (printf, 3, 4))); +#else +void REDISMODULE_API_FUNC(RedisModule_Log)(RedisModuleCtx *ctx, const char *level, const char *fmt, ...); +void REDISMODULE_API_FUNC(RedisModule_LogIOError)(RedisModuleIO *io, const char *levelstr, const char *fmt, ...); +#endif void REDISMODULE_API_FUNC(RedisModule__Assert)(const char *estr, const char *file, int line); void REDISMODULE_API_FUNC(RedisModule_LatencyAddSample)(const char *event, mstime_t latency); int REDISMODULE_API_FUNC(RedisModule_StringAppendBuffer)(RedisModuleCtx *ctx, RedisModuleString *str, const char *buf, size_t len); From 0529bc185a35341e34af60a044af6e48ed13c73e Mon Sep 17 00:00:00 2001 From: Guy Benoish Date: Thu, 2 Apr 2020 14:57:17 +0300 Subject: [PATCH 0245/1280] Try to fix time-sensitive tests in blockonkey.tcl There is an inherent race between the deferring client and the "main" client of the test: While the deferring client issues a blocking command, we can't know for sure that by the time the "main" client tries to issue another command (Usually one that unblocks the deferring client) the deferring client is even blocked... For lack of a better choice this commit uses TCL's 'after' in order to give some time for the deferring client to issues its blocking command before the "main" client does its thing. This problem probably exists in many other tests but this commit tries to fix blockonkeys.tcl --- tests/unit/moduleapi/blockonkeys.tcl | 55 +++++++++++++++++++++++++++- 1 file changed, 54 insertions(+), 1 deletion(-) diff --git a/tests/unit/moduleapi/blockonkeys.tcl b/tests/unit/moduleapi/blockonkeys.tcl index c8b8f23ed..5e5d93da3 100644 --- a/tests/unit/moduleapi/blockonkeys.tcl +++ b/tests/unit/moduleapi/blockonkeys.tcl @@ -11,6 +11,12 @@ start_server {tags {"modules"}} { $rd1 fsl.bpoppush src dst 0 $rd2 fsl.bpoppush dst src 0 + ;# wait until clients are actually blocked + wait_for_condition 50 100 { + [s 0 blocked_clients] eq {2} + } else { + fail "Clients are not blocked" + } r fsl.push src 42 @@ -24,7 +30,12 @@ start_server {tags {"modules"}} { r del src $rd1 fsl.bpoppush src src 0 - + ;# wait until clients are actually blocked + wait_for_condition 50 100 { + [s 0 blocked_clients] eq {1} + } else { + fail "Clients are not blocked" + } r fsl.push src 42 assert_equal {42} [r fsl.getall src] @@ -48,6 +59,12 @@ start_server {tags {"modules"}} { r del k set rd [redis_deferring_client] $rd fsl.bpop k 0 + ;# wait until clients are actually blocked + wait_for_condition 50 100 { + [s 0 blocked_clients] eq {1} + } else { + fail "Clients are not blocked" + } r fsl.push k 34 assert_equal {34} [$rd read] } @@ -76,6 +93,12 @@ start_server {tags {"modules"}} { set cid [$rd read] r fsl.push k 33 $rd fsl.bpopgt k 33 0 + ;# wait until clients are actually blocked + wait_for_condition 50 100 { + [s 0 blocked_clients] eq {1} + } else { + fail "Clients are not blocked" + } r fsl.push k 34 assert_equal {34} [$rd read] r client kill id $cid ;# try to smoke-out client-related memory leak @@ -85,6 +108,12 @@ start_server {tags {"modules"}} { r del k set rd [redis_deferring_client] $rd fsl.bpopgt k 35 0 + ;# wait until clients are actually blocked + wait_for_condition 50 100 { + [s 0 blocked_clients] eq {1} + } else { + fail "Clients are not blocked" + } r fsl.push k 33 r fsl.push k 34 r fsl.push k 35 @@ -98,6 +127,12 @@ start_server {tags {"modules"}} { $rd client id set cid [$rd read] $rd fsl.bpopgt k 35 0 + ;# wait until clients are actually blocked + wait_for_condition 50 100 { + [s 0 blocked_clients] eq {1} + } else { + fail "Clients are not blocked" + } r client kill id $cid ;# try to smoke-out client-related memory leak } @@ -107,6 +142,12 @@ start_server {tags {"modules"}} { $rd client id set cid [$rd read] $rd fsl.bpopgt k 35 0 + ;# wait until clients are actually blocked + wait_for_condition 50 100 { + [s 0 blocked_clients] eq {1} + } else { + fail "Clients are not blocked" + } r client unblock $cid timeout ;# try to smoke-out client-related memory leak assert_equal {Request timedout} [$rd read] } @@ -117,6 +158,12 @@ start_server {tags {"modules"}} { $rd client id set cid [$rd read] $rd fsl.bpopgt k 35 0 + ;# wait until clients are actually blocked + wait_for_condition 50 100 { + [s 0 blocked_clients] eq {1} + } else { + fail "Clients are not blocked" + } r client unblock $cid error ;# try to smoke-out client-related memory leak assert_error "*unblocked*" {$rd read} } @@ -125,6 +172,12 @@ start_server {tags {"modules"}} { r del k set rd [redis_deferring_client] $rd fsl.bpop k 0 + ;# wait until clients are actually blocked + wait_for_condition 50 100 { + [s 0 blocked_clients] eq {1} + } else { + fail "Clients are not blocked" + } r lpush k 12 r lpush k 13 r lpush k 14 From 98208c3f3068fa57155fec9a7beadbc3732a7ead Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Mon, 6 Apr 2020 09:41:14 +0300 Subject: [PATCH 0246/1280] diffrent fix for runtest --host --port --- tests/support/server.tcl | 7 ++----- tests/test_helper.tcl | 19 +++++++++++-------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/tests/support/server.tcl b/tests/support/server.tcl index 400017c5f..d086366dc 100644 --- a/tests/support/server.tcl +++ b/tests/support/server.tcl @@ -159,12 +159,9 @@ proc start_server {options {code undefined}} { if {$::external} { if {[llength $::servers] == 0} { set srv {} - # In test_server_main(tests/test_helper.tcl:215~218), increase the value of start_port - # and assign it to ::port through the `--port` option, so we need to reduce it. - set baseport [expr {$::port-100}] dict set srv "host" $::host - dict set srv "port" $baseport - set client [redis $::host $baseport 0 $::tls] + dict set srv "port" $::port + set client [redis $::host $::port 0 $::tls] dict set srv "client" $client $client select 9 diff --git a/tests/test_helper.tcl b/tests/test_helper.tcl index 5cb43104b..d80cb6907 100644 --- a/tests/test_helper.tcl +++ b/tests/test_helper.tcl @@ -212,13 +212,19 @@ proc test_server_main {} { # Start the client instances set ::clients_pids {} - set start_port [expr {$::port+100}] - for {set j 0} {$j < $::numclients} {incr j} { - set start_port [find_available_port $start_port] + if {$::external} { set p [exec $tclsh [info script] {*}$::argv \ - --client $port --port $start_port &] + --client $port --port $::port &] lappend ::clients_pids $p - incr start_port 10 + } else { + set start_port [expr {$::port+100}] + for {set j 0} {$j < $::numclients} {incr j} { + set start_port [find_available_port $start_port] + set p [exec $tclsh [info script] {*}$::argv \ + --client $port --port $start_port &] + lappend ::clients_pids $p + incr start_port 10 + } } # Setup global state for the test server @@ -506,9 +512,6 @@ for {set j 0} {$j < [llength $argv]} {incr j} { } elseif {$opt eq {--host}} { set ::external 1 set ::host $arg - # If we use an external server, we can only set numclients to 1, - # otherwise the port will be miscalculated. - set ::numclients 1 incr j } elseif {$opt eq {--port}} { set ::port $arg From 22905bae9861332b393cda449a0775446026a15d Mon Sep 17 00:00:00 2001 From: mymilkbottles Date: Mon, 6 Apr 2020 19:27:06 +0800 Subject: [PATCH 0247/1280] Judge the log level in advance --- src/scripting.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/scripting.c b/src/scripting.c index 32a511e13..bccbcf637 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -959,6 +959,7 @@ int luaLogCommand(lua_State *lua) { lua_pushstring(lua, "Invalid debug level."); return lua_error(lua); } + if (level < server.verbosity) return 0; /* Glue together all the arguments */ log = sdsempty(); From 73b3c87eb101b5cf4820481eb9f4885919a0943b Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 6 Apr 2020 13:23:07 +0200 Subject: [PATCH 0248/1280] LCS: get rid of STOREIDX option. Fix get keys helper. --- src/db.c | 14 +++++++------- src/t_string.c | 27 +++++++++++++-------------- 2 files changed, 20 insertions(+), 21 deletions(-) diff --git a/src/db.c b/src/db.c index bfa39564e..e9e10aeeb 100644 --- a/src/db.c +++ b/src/db.c @@ -1554,30 +1554,30 @@ int *georadiusGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numk return keys; } -/* LCS ... [STOREIDX ] ... */ +/* LCS ... [KEYS ] ... */ int *lcsGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numkeys) { int i; - int *keys; + int *keys = getKeysTempBuffer; UNUSED(cmd); /* We need to parse the options of the command in order to check for the - * "STOREIDX" argument before the STRINGS argument. */ + * "KEYS" argument before the "STRINGS" argument. */ for (i = 1; i < argc; i++) { char *arg = argv[i]->ptr; int moreargs = (argc-1) - i; if (!strcasecmp(arg, "strings")) { break; - } else if (!strcasecmp(arg, "storeidx") && moreargs) { - keys = getKeysTempBuffer; + } else if (!strcasecmp(arg, "keys") && moreargs >= 2) { keys[0] = i+1; - *numkeys = 1; + keys[1] = i+2; + *numkeys = 2; return keys; } } *numkeys = 0; - return NULL; + return keys; } /* Helper function to extract keys from memory command. diff --git a/src/t_string.c b/src/t_string.c index 80cdbb5e1..deba22253 100644 --- a/src/t_string.c +++ b/src/t_string.c @@ -482,14 +482,13 @@ void strlenCommand(client *c) { /* LCS -- Longest common subsequence. * - * LCS [IDX] [STOREIDX ] [MINMATCHLEN ] + * LCS [IDX] [MINMATCHLEN ] * STRINGS | KEYS */ void lcsCommand(client *c) { uint32_t i, j; long long minmatchlen = 0; sds a = NULL, b = NULL; int getlen = 0, getidx = 0, withmatchlen = 0; - robj *idxkey = NULL; /* STOREIDX will set this and getidx to 1. */ robj *obja = NULL, *objb = NULL; for (j = 1; j < (uint32_t)c->argc; j++) { @@ -502,17 +501,16 @@ void lcsCommand(client *c) { getlen = 1; } else if (!strcasecmp(opt,"WITHMATCHLEN")) { withmatchlen = 1; - } else if (!strcasecmp(opt,"STOREIDX") && moreargs) { - getidx = 1; - idxkey = c->argv[j+1]; - j++; } else if (!strcasecmp(opt,"MINMATCHLEN") && moreargs) { if (getLongLongFromObjectOrReply(c,c->argv[j+1],&minmatchlen,NULL) != C_OK) return; if (minmatchlen < 0) minmatchlen = 0; j++; } else if (!strcasecmp(opt,"STRINGS")) { - if (moreargs != 2) { + if (a != NULL) { + addReplyError(c,"Either use STRINGS or KEYS"); + return; + } else if (moreargs != 2) { addReplyError(c,"LCS requires exactly two strings"); return; } @@ -520,7 +518,10 @@ void lcsCommand(client *c) { b = c->argv[j+2]->ptr; j += 2; } else if (!strcasecmp(opt,"KEYS")) { - if (moreargs != 2) { + if (a != NULL) { + addReplyError(c,"Either use STRINGS or KEYS"); + return; + } else if (moreargs != 2) { addReplyError(c,"LCS requires exactly two keys"); return; } @@ -542,12 +543,10 @@ void lcsCommand(client *c) { addReplyError(c,"Please specify two strings: " "STRINGS or KEYS options are mandatory"); return; - } else if (getlen && (getidx && idxkey == NULL)) { + } else if (getlen && getidx) { addReplyError(c, - "If you want both the LEN and indexes, please " - "store the indexes into a key with STOREIDX. However note " - "that the IDX output also includes both the LCS string and " - "its length"); + "If you want both the length and indexes, please " + "just use IDX."); return; } @@ -602,7 +601,7 @@ void lcsCommand(client *c) { /* Start with a deferred array if we have to emit the ranges. */ uint32_t arraylen = 0; /* Number of ranges emitted in the array. */ - if (getidx && idxkey == NULL) { + if (getidx) { addReplyMapLen(c,2); addReplyBulkCString(c,"matches"); arraylenptr = addReplyDeferredLen(c); From 2f1c1ffa0bcb4d2010138d67ebf42ab7f584205c Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 6 Apr 2020 13:45:37 +0200 Subject: [PATCH 0249/1280] LCS tests. --- tests/unit/type/string.tcl | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/tests/unit/type/string.tcl b/tests/unit/type/string.tcl index 7122fd987..131a80ad0 100644 --- a/tests/unit/type/string.tcl +++ b/tests/unit/type/string.tcl @@ -419,4 +419,26 @@ start_server {tags {"string"}} { r set foo bar r getrange foo 0 4294967297 } {bar} + + set rna1 {CACCTTCCCAGGTAACAAACCAACCAACTTTCGATCTCTTGTAGATCTGTTCTCTAAACGAACTTTAAAATCTGTGTGGCTGTCACTCGGCTGCATGCTTAGTGCACTCACGCAGTATAATTAATAACTAATTACTGTCGTTGACAGGACACGAGTAACTCGTCTATCTTCTGCAGGCTGCTTACGGTTTCGTCCGTGTTGCAGCCGATCATCAGCACATCTAGGTTTCGTCCGGGTGTG} + set rna2 {ATTAAAGGTTTATACCTTCCCAGGTAACAAACCAACCAACTTTCGATCTCTTGTAGATCTGTTCTCTAAACGAACTTTAAAATCTGTGTGGCTGTCACTCGGCTGCATGCTTAGTGCACTCACGCAGTATAATTAATAACTAATTACTGTCGTTGACAGGACACGAGTAACTCGTCTATCTTCTGCAGGCTGCTTACGGTTTCGTCCGTGTTGCAGCCGATCATCAGCACATCTAGGTTT} + set rnalcs {ACCTTCCCAGGTAACAAACCAACCAACTTTCGATCTCTTGTAGATCTGTTCTCTAAACGAACTTTAAAATCTGTGTGGCTGTCACTCGGCTGCATGCTTAGTGCACTCACGCAGTATAATTAATAACTAATTACTGTCGTTGACAGGACACGAGTAACTCGTCTATCTTCTGCAGGCTGCTTACGGTTTCGTCCGTGTTGCAGCCGATCATCAGCACATCTAGGTTT} + + test {LCS string output with STRINGS option} { + r LCS STRINGS $rna1 $rna2 + } $rnalcs + + test {LCS len} { + r LCS LEN STRINGS $rna1 $rna2 + } [string length $rnalcs] + + test {LCS with KEYS option} { + r set virus1 $rna1 + r set virus2 $rna2 + r LCS KEYS virus1 virus2 + } $rnalcs + + test {LCS indexes} { + dict get [r LCS IDX KEYS virus1 virus2] matches + } {{{238 238} {239 239}} {{236 236} {238 238}} {{229 230} {236 237}} {{224 224} {235 235}} {{1 222} {13 234}}} } From d48eb1bbe50e552fcf1a16e5bb881bd0e1f19edd Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 6 Apr 2020 13:48:31 +0200 Subject: [PATCH 0250/1280] LCS: allow KEYS / STRINGS to be anywhere. Initially they needed to be at the end so that we could extend to N strings in the future, but after further consideration I no longer believe it's worth it. --- src/t_string.c | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/t_string.c b/src/t_string.c index deba22253..4e693cefa 100644 --- a/src/t_string.c +++ b/src/t_string.c @@ -510,9 +510,6 @@ void lcsCommand(client *c) { if (a != NULL) { addReplyError(c,"Either use STRINGS or KEYS"); return; - } else if (moreargs != 2) { - addReplyError(c,"LCS requires exactly two strings"); - return; } a = c->argv[j+1]->ptr; b = c->argv[j+2]->ptr; @@ -521,9 +518,6 @@ void lcsCommand(client *c) { if (a != NULL) { addReplyError(c,"Either use STRINGS or KEYS"); return; - } else if (moreargs != 2) { - addReplyError(c,"LCS requires exactly two keys"); - return; } obja = lookupKeyRead(c->db,c->argv[j+1]); objb = lookupKeyRead(c->db,c->argv[j+2]); From 9cd8e2d749b1a09bc26736118ed4c1175c80e52a Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 6 Apr 2020 13:51:49 +0200 Subject: [PATCH 0251/1280] LCS: more tests. --- tests/unit/type/string.tcl | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/unit/type/string.tcl b/tests/unit/type/string.tcl index 131a80ad0..b9ef9de7a 100644 --- a/tests/unit/type/string.tcl +++ b/tests/unit/type/string.tcl @@ -441,4 +441,12 @@ start_server {tags {"string"}} { test {LCS indexes} { dict get [r LCS IDX KEYS virus1 virus2] matches } {{{238 238} {239 239}} {{236 236} {238 238}} {{229 230} {236 237}} {{224 224} {235 235}} {{1 222} {13 234}}} + + test {LCS indexes with match len} { + dict get [r LCS IDX KEYS virus1 virus2 WITHMATCHLEN] matches + } {{{238 238} {239 239} 1} {{236 236} {238 238} 1} {{229 230} {236 237} 2} {{224 224} {235 235} 1} {{1 222} {13 234} 222}} + + test {LCS indexes with match len and minimum match len} { + dict get [r LCS IDX KEYS virus1 virus2 WITHMATCHLEN MINMATCHLEN 5] matches + } {{{1 222} {13 234} 222}} } From 84979cadf637d000d793124eb87be6407a2fb105 Mon Sep 17 00:00:00 2001 From: qetu3790 Date: Mon, 6 Apr 2020 20:52:32 +0800 Subject: [PATCH 0252/1280] fix comments about RESIZE DB opcode in rdb.c fix comments about RESIZE DB opcode in rdb.c --- src/rdb.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/rdb.c b/src/rdb.c index fc8911979..78ec83cce 100644 --- a/src/rdb.c +++ b/src/rdb.c @@ -1175,10 +1175,7 @@ int rdbSaveRio(rio *rdb, int *error, int rdbflags, rdbSaveInfo *rsi) { if (rdbSaveType(rdb,RDB_OPCODE_SELECTDB) == -1) goto werr; if (rdbSaveLen(rdb,j) == -1) goto werr; - /* Write the RESIZE DB opcode. We trim the size to UINT32_MAX, which - * is currently the largest type we are able to represent in RDB sizes. - * However this does not limit the actual size of the DB to load since - * these sizes are just hints to resize the hash tables. */ + /* Write the RESIZE DB opcode. */ uint64_t db_size, expires_size; db_size = dictSize(db->dict); expires_size = dictSize(db->expires); From 1bbe60eab88e6c02a92ecedface5360cd6687ef7 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 6 Apr 2020 17:32:04 +0200 Subject: [PATCH 0253/1280] Clarify redis.conf comment about lazyfree-lazy-user-del. --- redis.conf | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/redis.conf b/redis.conf index 14388e320..5baeae65f 100644 --- a/redis.conf +++ b/redis.conf @@ -936,23 +936,27 @@ replica-priority 100 # or SORT with STORE option may delete existing keys. The SET command # itself removes any old content of the specified key in order to replace # it with the specified string. -# 4) The DEL command itself, and normally it's not easy to replace DEL with -# UNLINK in user's codes. -# 5) During replication, when a replica performs a full resynchronization with +# 4) During replication, when a replica performs a full resynchronization with # its master, the content of the whole database is removed in order to # load the RDB file just transferred. # # In all the above cases the default is to delete objects in a blocking way, # like if DEL was called. However you can configure each case specifically # in order to instead release memory in a non-blocking way like if UNLINK -# was called, using the following configuration directives: +# was called, using the following configuration directives. lazyfree-lazy-eviction no lazyfree-lazy-expire no lazyfree-lazy-server-del no -lazyfree-lazy-user-del no replica-lazy-flush no +# It is also possible, for the case when to replace the user code DEL calls +# with UNLINK calls is not easy, to modify the default behavior of the DEL +# command to act exactly like UNLINK, using the following configuration +# directive: + +lazyfree-lazy-user-del no + ################################ THREADED I/O ################################# # Redis is mostly single threaded, however there are certain threaded From 45d7fcec222b10fd762ef23ee3b0b22d18b0c356 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 7 Apr 2020 12:07:09 +0200 Subject: [PATCH 0254/1280] Speedup INFO by counting client memory incrementally. Related to #5145. Design note: clients may change type when they turn into replicas or are moved into the Pub/Sub category and so forth. Moreover the recomputation of the bytes used is problematic for obvious reasons: it changes continuously, so as a conservative way to avoid accumulating errors, each client remembers the contribution it gave to the sum, and removes it when it is freed or before updating it with the new memory usage. --- src/networking.c | 7 +++++++ src/object.c | 33 +++++++++------------------------ src/server.c | 25 +++++++++++++++++++++++++ src/server.h | 13 +++++++++++-- 4 files changed, 52 insertions(+), 26 deletions(-) diff --git a/src/networking.c b/src/networking.c index 85c640e34..654fda517 100644 --- a/src/networking.c +++ b/src/networking.c @@ -157,6 +157,8 @@ client *createClient(connection *conn) { c->client_list_node = NULL; c->client_tracking_redirection = 0; c->client_tracking_prefixes = NULL; + c->client_cron_last_memory_usage = 0; + c->client_cron_last_memory_type = CLIENT_TYPE_NORMAL; c->auth_callback = NULL; c->auth_callback_privdata = NULL; c->auth_module = NULL; @@ -1160,6 +1162,11 @@ void freeClient(client *c) { listDelNode(server.clients_to_close,ln); } + /* Remove the contribution that this client gave to our + * incrementally computed memory usage. */ + server.stat_clients_type_memory[c->client_cron_last_memory_type] -= + c->client_cron_last_memory_usage; + /* Release other dynamically allocated client structure fields, * and finally release the client structure itself. */ if (c->name) decrRefCount(c->name); diff --git a/src/object.c b/src/object.c index 11e335afc..52d5b11f5 100644 --- a/src/object.c +++ b/src/object.c @@ -974,30 +974,15 @@ struct redisMemOverhead *getMemoryOverheadData(void) { mh->repl_backlog = mem; mem_total += mem; - mem = 0; - if (listLength(server.clients)) { - listIter li; - listNode *ln; - size_t mem_normal = 0, mem_slaves = 0; - - listRewind(server.clients,&li); - while((ln = listNext(&li))) { - size_t mem_curr = 0; - client *c = listNodeValue(ln); - int type = getClientType(c); - mem_curr += getClientOutputBufferMemoryUsage(c); - mem_curr += sdsAllocSize(c->querybuf); - mem_curr += sizeof(client); - if (type == CLIENT_TYPE_SLAVE) - mem_slaves += mem_curr; - else - mem_normal += mem_curr; - } - mh->clients_slaves = mem_slaves; - mh->clients_normal = mem_normal; - mem = mem_slaves + mem_normal; - } - mem_total+=mem; + /* Computing the memory used by the clients would be O(N) if done + * here online. We use our values computed incrementally by + * clientsCronTrackClientsMemUsage(). */ + mh->clients_slaves = server.stat_clients_type_memory[CLIENT_TYPE_SLAVE]; + mh->clients_normal = server.stat_clients_type_memory[CLIENT_TYPE_MASTER]+ + server.stat_clients_type_memory[CLIENT_TYPE_PUBSUB]+ + server.stat_clients_type_memory[CLIENT_TYPE_NORMAL]; + mem_total += mh->clients_slaves; + mem_total += mh->clients_normal; mem = 0; if (server.aof_state != AOF_OFF) { diff --git a/src/server.c b/src/server.c index 56feb09a3..996e0f5d2 100644 --- a/src/server.c +++ b/src/server.c @@ -1593,6 +1593,28 @@ int clientsCronTrackExpansiveClients(client *c) { return 0; /* This function never terminates the client. */ } +/* Iterating all the clients in getMemoryOverheadData() is too slow and + * in turn would make the INFO command too slow. So we perform this + * computation incrementally and track the (not instantaneous but updated + * to the second) total memory used by clients using clinetsCron() in + * a more incremental way (depending on server.hz). */ +int clientsCronTrackClientsMemUsage(client *c) { + size_t mem = 0; + int type = getClientType(c); + mem += getClientOutputBufferMemoryUsage(c); + mem += sdsAllocSize(c->querybuf); + mem += sizeof(client); + /* Now that we have the memory used by the client, remove the old + * value from the old categoty, and add it back. */ + server.stat_clients_type_memory[c->client_cron_last_memory_type] -= + c->client_cron_last_memory_usage; + server.stat_clients_type_memory[type] += mem; + /* Remember what we added and where, to remove it next time. */ + c->client_cron_last_memory_usage = mem; + c->client_cron_last_memory_type = type; + return 0; +} + /* Return the max samples in the memory usage of clients tracked by * the function clientsCronTrackExpansiveClients(). */ void getExpansiveClientsInfo(size_t *in_usage, size_t *out_usage) { @@ -1653,6 +1675,7 @@ void clientsCron(void) { if (clientsCronHandleTimeout(c,now)) continue; if (clientsCronResizeQueryBuffer(c)) continue; if (clientsCronTrackExpansiveClients(c)) continue; + if (clientsCronTrackClientsMemUsage(c)) continue; } } @@ -2792,6 +2815,8 @@ void initServer(void) { server.stat_rdb_cow_bytes = 0; server.stat_aof_cow_bytes = 0; server.stat_module_cow_bytes = 0; + for (int j = 0; j < CLIENT_TYPE_COUNT; j++) + server.stat_clients_type_memory[j] = 0; server.cron_malloc_stats.zmalloc_used = 0; server.cron_malloc_stats.process_rss = 0; server.cron_malloc_stats.allocator_allocated = 0; diff --git a/src/server.h b/src/server.h index 9b77f55ac..cf4c285f8 100644 --- a/src/server.h +++ b/src/server.h @@ -274,6 +274,7 @@ typedef long long ustime_t; /* microsecond time type. */ #define CLIENT_TYPE_SLAVE 1 /* Slaves. */ #define CLIENT_TYPE_PUBSUB 2 /* Clients subscribed to PubSub channels. */ #define CLIENT_TYPE_MASTER 3 /* Master. */ +#define CLIENT_TYPE_COUNT 4 /* Total number of client types. */ #define CLIENT_TYPE_OBUF_COUNT 3 /* Number of clients to expose to output buffer configuration. Just the first three: normal, slave, pubsub. */ @@ -820,10 +821,10 @@ typedef struct client { * when the authenticated user * changes. */ void *auth_callback_privdata; /* Private data that is passed when the auth - * changed callback is executed. Opaque for + * changed callback is executed. Opaque for * Redis Core. */ void *auth_module; /* The module that owns the callback, which is used - * to disconnect the client if the module is + * to disconnect the client if the module is * unloaded for cleanup. Opaque for Redis Core.*/ /* If this client is in tracking mode and this field is non zero, @@ -833,6 +834,13 @@ typedef struct client { rax *client_tracking_prefixes; /* A dictionary of prefixes we are already subscribed to in BCAST mode, in the context of client side caching. */ + /* In clientsCronTrackClientsMemUsage() we track the memory usage of + * each client and add it to the sum of all the clients of a given type, + * however we need to remember what was the old contribution of each + * client, and in which categoty the client was, in order to remove it + * before adding it the new value. */ + uint64_t client_cron_last_memory_usage; + int client_cron_last_memory_type; /* Response buffer */ int bufpos; char buf[PROTO_REPLY_CHUNK_BYTES]; @@ -1129,6 +1137,7 @@ struct redisServer { size_t stat_rdb_cow_bytes; /* Copy on write bytes during RDB saving. */ size_t stat_aof_cow_bytes; /* Copy on write bytes during AOF rewrite. */ size_t stat_module_cow_bytes; /* Copy on write bytes during module fork. */ + uint64_t stat_clients_type_memory[CLIENT_TYPE_COUNT];/* Mem usage by type */ long long stat_unexpected_error_replies; /* Number of unexpected (aof-loading, replica to master, etc.) error replies */ /* The following two are used to track instantaneous metrics, like * number of operations per second, network traffic. */ From 9a604033dd6445bc229807e32b9ea73ed686a9fb Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 8 Apr 2020 10:54:18 +0200 Subject: [PATCH 0255/1280] Fix ACL HELP table missing comma. --- src/acl.c | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/acl.c b/src/acl.c index 27f4bdb84..733988013 100644 --- a/src/acl.c +++ b/src/acl.c @@ -1847,18 +1847,18 @@ void aclCommand(client *c) { } } else if (!strcasecmp(sub,"help")) { const char *help[] = { -"LOAD -- Reload users from the ACL file.", -"SAVE -- Save the current config to the ACL file." -"LIST -- Show user details in config file format.", -"USERS -- List all the registered usernames.", -"SETUSER [attribs ...] -- Create or modify a user.", -"GETUSER -- Get the user details.", -"DELUSER [...] -- Delete a list of users.", -"CAT -- List available categories.", -"CAT -- List commands inside category.", -"GENPASS -- Generate a secure user password.", -"WHOAMI -- Return the current connection username.", -"LOG [ | RESET] -- Show the ACL log entries.", +"LOAD -- Reload users from the ACL file.", +"SAVE -- Save the current config to the ACL file.", +"LIST -- Show user details in config file format.", +"USERS -- List all the registered usernames.", +"SETUSER [attribs ...] -- Create or modify a user.", +"GETUSER -- Get the user details.", +"DELUSER [...] -- Delete a list of users.", +"CAT -- List available categories.", +"CAT -- List commands inside category.", +"GENPASS -- Generate a secure user password.", +"WHOAMI -- Return the current connection username.", +"LOG [ | RESET] -- Show the ACL log entries.", NULL }; addReplyHelp(c,help); From 4406928a9036a2e402ef6619310c4aa45b13eb0d Mon Sep 17 00:00:00 2001 From: hayleeliu Date: Wed, 8 Apr 2020 18:20:32 +0800 Subject: [PATCH 0256/1280] fix spelling mistake in bitops.c --- src/bitops.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bitops.c b/src/bitops.c index f78e4fd34..496d78b66 100644 --- a/src/bitops.c +++ b/src/bitops.c @@ -269,7 +269,7 @@ int64_t getSignedBitfield(unsigned char *p, uint64_t offset, uint64_t bits) { * then zero is returned, otherwise in case of overflow, 1 is returned, * otherwise in case of underflow, -1 is returned. * - * When non-zero is returned (oferflow or underflow), if not NULL, *limit is + * When non-zero is returned (overflow or underflow), if not NULL, *limit is * set to the value the operation should result when an overflow happens, * depending on the specified overflow semantics: * From a27cd352d3c25de4a63d95147ff657edf6b804e0 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 8 Apr 2020 12:55:57 +0200 Subject: [PATCH 0257/1280] Speedup: unblock clients on keys in O(1). See #7071. --- src/adlist.c | 20 +++++++++++++++++--- src/adlist.h | 3 ++- src/blocked.c | 48 ++++++++++++++++++++++++++++++------------------ src/server.c | 2 +- 4 files changed, 50 insertions(+), 23 deletions(-) diff --git a/src/adlist.c b/src/adlist.c index ec5f8bbf4..0fedc0729 100644 --- a/src/adlist.c +++ b/src/adlist.c @@ -327,12 +327,11 @@ listNode *listIndex(list *list, long index) { } /* Rotate the list removing the tail node and inserting it to the head. */ -void listRotate(list *list) { - listNode *tail = list->tail; - +void listRotateTailToHead(list *list) { if (listLength(list) <= 1) return; /* Detach current tail */ + listNode *tail = list->tail; list->tail = tail->prev; list->tail->next = NULL; /* Move it as head */ @@ -342,6 +341,21 @@ void listRotate(list *list) { list->head = tail; } +/* Rotate the list removing the head node and inserting it to the tail. */ +void listRotateHeadToTail(list *list) { + if (listLength(list) <= 1) return; + + listNode *head = list->head; + /* Detach current head */ + list->head = head->next; + list->head->prev = NULL; + /* Move it as tail */ + list->tail->next = head; + head->next = NULL; + head->prev = list->tail; + list->tail = head; +} + /* Add all the elements of the list 'o' at the end of the * list 'l'. The list 'other' remains empty but otherwise valid. */ void listJoin(list *l, list *o) { diff --git a/src/adlist.h b/src/adlist.h index 28b9016ce..dd8a8d693 100644 --- a/src/adlist.h +++ b/src/adlist.h @@ -85,7 +85,8 @@ listNode *listSearchKey(list *list, void *key); listNode *listIndex(list *list, long index); void listRewind(list *list, listIter *li); void listRewindTail(list *list, listIter *li); -void listRotate(list *list); +void listRotateTailToHead(list *list); +void listRotateHeadToTail(list *list); void listJoin(list *l, list *o); /* Directions for iterators */ diff --git a/src/blocked.c b/src/blocked.c index e3a803ae3..00cc798d5 100644 --- a/src/blocked.c +++ b/src/blocked.c @@ -64,6 +64,21 @@ int serveClientBlockedOnList(client *receiver, robj *key, robj *dstkey, redisDb *db, robj *value, int where); +/* This structure represents the blocked key information that we store + * in the client structure. Each client blocked on keys, has a + * client->bpop.keys hash table. The keys of the hash table are Redis + * keys pointers to 'robj' structures. The value is this structure. + * The structure has two goals: firstly we store the list node that this + * client uses to be listed in the database "blocked clients for this key" + * list, so we can later unblock in O(1) without a list scan. + * Secondly for certain blocking types, we have additional info. Right now + * the only use for additional info we have is when clients are blocked + * on streams, as we have to remember the ID it blocked for. */ +typedef struct bkinfo { + listNode *listnode; /* List node for db->blocking_keys[key] list. */ + streamID stream_id; /* Stream ID if we blocked in a stream. */ +} bkinfo; + /* Block a client for the specific operation type. Once the CLIENT_BLOCKED * flag is set client query buffer is not longer processed, but accumulated, * and will be processed when the client is unblocked. */ @@ -211,8 +226,7 @@ void serveClientsBlockedOnListKey(robj *o, readyList *rl) { if (receiver->btype != BLOCKED_LIST) { /* Put at the tail, so that at the next call * we'll not run into it again. */ - listDelNode(clients,clientnode); - listAddNodeTail(clients,receiver); + listRotateHeadToTail(clients); continue; } @@ -273,8 +287,7 @@ void serveClientsBlockedOnSortedSetKey(robj *o, readyList *rl) { if (receiver->btype != BLOCKED_ZSET) { /* Put at the tail, so that at the next call * we'll not run into it again. */ - listDelNode(clients,clientnode); - listAddNodeTail(clients,receiver); + listRotateHeadToTail(clients); continue; } @@ -320,8 +333,8 @@ void serveClientsBlockedOnStreamKey(robj *o, readyList *rl) { while((ln = listNext(&li))) { client *receiver = listNodeValue(ln); if (receiver->btype != BLOCKED_STREAM) continue; - streamID *gt = dictFetchValue(receiver->bpop.keys, - rl->key); + bkinfo *bki = dictFetchValue(receiver->bpop.keys,rl->key); + streamID *gt = &bki->stream_id; /* If we blocked in the context of a consumer * group, we need to resolve the group and update the @@ -419,8 +432,7 @@ void serveClientsBlockedOnKeyByModule(readyList *rl) { * ready to be served, so they'll remain in the list * sometimes. We want also be able to skip clients that are * not blocked for the MODULE type safely. */ - listDelNode(clients,clientnode); - listAddNodeTail(clients,receiver); + listRotateHeadToTail(clients); if (receiver->btype != BLOCKED_MODULE) continue; @@ -551,17 +563,15 @@ void blockForKeys(client *c, int btype, robj **keys, int numkeys, mstime_t timeo if (target != NULL) incrRefCount(target); for (j = 0; j < numkeys; j++) { - /* The value associated with the key name in the bpop.keys dictionary - * is NULL for lists and sorted sets, or the stream ID for streams. */ - void *key_data = NULL; - if (btype == BLOCKED_STREAM) { - key_data = zmalloc(sizeof(streamID)); - memcpy(key_data,ids+j,sizeof(streamID)); - } + /* Allocate our bkinfo structure, associated to each key the client + * is blocked for. */ + bkinfo *bki = zmalloc(sizeof(*bki)); + if (btype == BLOCKED_STREAM) + bki->stream_id = ids[j]; /* If the key already exists in the dictionary ignore it. */ - if (dictAdd(c->bpop.keys,keys[j],key_data) != DICT_OK) { - zfree(key_data); + if (dictAdd(c->bpop.keys,keys[j],bki) != DICT_OK) { + zfree(bki); continue; } incrRefCount(keys[j]); @@ -580,6 +590,7 @@ void blockForKeys(client *c, int btype, robj **keys, int numkeys, mstime_t timeo l = dictGetVal(de); } listAddNodeTail(l,c); + bki->listnode = listLast(l); } blockClient(c,btype); } @@ -596,11 +607,12 @@ void unblockClientWaitingData(client *c) { /* The client may wait for multiple keys, so unblock it for every key. */ while((de = dictNext(di)) != NULL) { robj *key = dictGetKey(de); + bkinfo *bki = dictGetVal(de); /* Remove this client from the list of clients waiting for this key. */ l = dictFetchValue(c->db->blocking_keys,key); serverAssertWithInfo(c,key,l != NULL); - listDelNode(l,listSearchKey(l,c)); + listDelNode(l,bki->listnode); /* If the list is empty we need to remove it to avoid wasting memory */ if (listLength(l) == 0) dictDelete(c->db->blocking_keys,key); diff --git a/src/server.c b/src/server.c index 996e0f5d2..0afd67514 100644 --- a/src/server.c +++ b/src/server.c @@ -1666,7 +1666,7 @@ void clientsCron(void) { /* Rotate the list, take the current head, process. * This way if the client must be removed from the list it's the * first element and we don't incur into O(N) computation. */ - listRotate(server.clients); + listRotateTailToHead(server.clients); head = listFirst(server.clients); c = listNodeValue(head); /* The following functions do different service checks on the client. From f9c0953fdd16d6d923e64629a15966e51c766d6c Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 9 Apr 2020 10:24:10 +0200 Subject: [PATCH 0258/1280] RDB: load files faster avoiding useless free+realloc. Reloading of the RDB generated by DEBUG POPULATE 5000000 SAVE is now 25% faster. This commit also prepares the ability to have more flexibility when loading stuff from the RDB, since we no longer use dbAdd() but can control exactly how things are added in the database. --- src/cluster.c | 2 +- src/db.c | 16 ++++++++-------- src/lazyfree.c | 2 +- src/rdb.c | 40 ++++++++++++++++++++++++++-------------- src/rdb.h | 2 +- src/redis-check-rdb.c | 2 +- src/server.h | 4 ++-- 7 files changed, 40 insertions(+), 28 deletions(-) diff --git a/src/cluster.c b/src/cluster.c index 385ff5763..2377b386b 100644 --- a/src/cluster.c +++ b/src/cluster.c @@ -4966,7 +4966,7 @@ void restoreCommand(client *c) { rioInitWithBuffer(&payload,c->argv[3]->ptr); if (((type = rdbLoadObjectType(&payload)) == -1) || - ((obj = rdbLoadObject(type,&payload,c->argv[1])) == NULL)) + ((obj = rdbLoadObject(type,&payload,c->argv[1]->ptr)) == NULL)) { addReplyError(c,"Bad data format"); return; diff --git a/src/db.c b/src/db.c index 9b5d62f29..d393a5fdd 100644 --- a/src/db.c +++ b/src/db.c @@ -185,7 +185,7 @@ void dbAdd(redisDb *db, robj *key, robj *val) { val->type == OBJ_ZSET || val->type == OBJ_STREAM) signalKeyAsReady(db, key); - if (server.cluster_enabled) slotToKeyAdd(key); + if (server.cluster_enabled) slotToKeyAdd(key->ptr); } /* Overwrite an existing key with a new value. Incrementing the reference @@ -288,7 +288,7 @@ int dbSyncDelete(redisDb *db, robj *key) { * the key, because it is shared with the main dictionary. */ if (dictSize(db->expires) > 0) dictDelete(db->expires,key->ptr); if (dictDelete(db->dict,key->ptr) == DICT_OK) { - if (server.cluster_enabled) slotToKeyDel(key); + if (server.cluster_enabled) slotToKeyDel(key->ptr); return 1; } else { return 0; @@ -1647,17 +1647,17 @@ int *xreadGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numkeys) * a fast way a key that belongs to a specified hash slot. This is useful * while rehashing the cluster and in other conditions when we need to * understand if we have keys for a given hash slot. */ -void slotToKeyUpdateKey(robj *key, int add) { - unsigned int hashslot = keyHashSlot(key->ptr,sdslen(key->ptr)); +void slotToKeyUpdateKey(sds key, int add) { + size_t keylen = sdslen(key); + unsigned int hashslot = keyHashSlot(key,keylen); unsigned char buf[64]; unsigned char *indexed = buf; - size_t keylen = sdslen(key->ptr); server.cluster->slots_keys_count[hashslot] += add ? 1 : -1; if (keylen+2 > 64) indexed = zmalloc(keylen+2); indexed[0] = (hashslot >> 8) & 0xff; indexed[1] = hashslot & 0xff; - memcpy(indexed+2,key->ptr,keylen); + memcpy(indexed+2,key,keylen); if (add) { raxInsert(server.cluster->slots_to_keys,indexed,keylen+2,NULL,NULL); } else { @@ -1666,11 +1666,11 @@ void slotToKeyUpdateKey(robj *key, int add) { if (indexed != buf) zfree(indexed); } -void slotToKeyAdd(robj *key) { +void slotToKeyAdd(sds key) { slotToKeyUpdateKey(key,1); } -void slotToKeyDel(robj *key) { +void slotToKeyDel(sds key) { slotToKeyUpdateKey(key,0); } diff --git a/src/lazyfree.c b/src/lazyfree.c index 3d3159c90..f01504e70 100644 --- a/src/lazyfree.c +++ b/src/lazyfree.c @@ -83,7 +83,7 @@ int dbAsyncDelete(redisDb *db, robj *key) { * field to NULL in order to lazy free it later. */ if (de) { dictFreeUnlinkedEntry(db->dict,de); - if (server.cluster_enabled) slotToKeyDel(key); + if (server.cluster_enabled) slotToKeyDel(key->ptr); return 1; } else { return 0; diff --git a/src/rdb.c b/src/rdb.c index 78ec83cce..d3104ffbf 100644 --- a/src/rdb.c +++ b/src/rdb.c @@ -1422,7 +1422,7 @@ robj *rdbLoadCheckModuleValue(rio *rdb, char *modulename) { /* Load a Redis object of the specified type from the specified file. * On success a newly allocated object is returned, otherwise NULL. */ -robj *rdbLoadObject(int rdbtype, rio *rdb, robj *key) { +robj *rdbLoadObject(int rdbtype, rio *rdb, sds key) { robj *o = NULL, *ele, *dec; uint64_t len; unsigned int i; @@ -1886,7 +1886,9 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, robj *key) { exit(1); } RedisModuleIO io; - moduleInitIOContext(io,mt,rdb,key); + robj keyobj; + initStaticStringObject(keyobj,key); + moduleInitIOContext(io,mt,rdb,&keyobj); io.ver = (rdbtype == RDB_TYPE_MODULE) ? 1 : 2; /* Call the rdb_load method of the module providing the 10 bit * encoding version in the lower 10 bits of the module ID. */ @@ -2044,7 +2046,8 @@ int rdbLoadRio(rio *rdb, int rdbflags, rdbSaveInfo *rsi) { long long lru_clock = LRU_CLOCK(); while(1) { - robj *key, *val; + sds key; + robj *val; /* Read type. */ if ((type = rdbLoadType(rdb)) == -1) goto eoferr; @@ -2216,10 +2219,11 @@ int rdbLoadRio(rio *rdb, int rdbflags, rdbSaveInfo *rsi) { } /* Read key */ - if ((key = rdbLoadStringObject(rdb)) == NULL) goto eoferr; + if ((key = rdbGenericLoadStringObject(rdb,RDB_LOAD_SDS,NULL)) == NULL) + goto eoferr; /* Read value */ if ((val = rdbLoadObject(type,rdb,key)) == NULL) { - decrRefCount(key); + sdsfree(key); goto eoferr; } @@ -2229,24 +2233,32 @@ int rdbLoadRio(rio *rdb, int rdbflags, rdbSaveInfo *rsi) { * responsible for key expiry. If we would expire keys here, the * snapshot taken by the master may not be reflected on the slave. */ if (iAmMaster() && !(rdbflags&RDBFLAGS_AOF_PREAMBLE) && expiretime != -1 && expiretime < now) { - decrRefCount(key); + sdsfree(key); decrRefCount(val); } else { /* Add the new object in the hash table */ - dbAdd(db,key,val); + int retval = dictAdd(db->dict, key, val); + if (retval != DICT_OK) { + serverLog(LL_WARNING, + "RDB has duplicated key '%s' in DB %d",key,db->id); + serverPanic("Duplicated key found in RDB file"); + } + if (server.cluster_enabled) slotToKeyAdd(key); /* Set the expire time if needed */ - if (expiretime != -1) setExpire(NULL,db,key,expiretime); + if (expiretime != -1) { + robj keyobj; + initStaticStringObject(keyobj,key); + setExpire(NULL,db,&keyobj,expiretime); + } /* Set usage information (for eviction). */ objectSetLRUOrLFU(val,lfu_freq,lru_idle,lru_clock,1000); - - /* Decrement the key refcount since dbAdd() will take its - * own reference. */ - decrRefCount(key); } - if (server.key_load_delay) - usleep(server.key_load_delay); + + /* Loading the database more slowly is useful in order to test + * certain edge cases. */ + if (server.key_load_delay) usleep(server.key_load_delay); /* Reset the state that is key-specified and is populated by * opcodes before the key, so that we start from scratch again. */ diff --git a/src/rdb.h b/src/rdb.h index b276a978b..526ed116b 100644 --- a/src/rdb.h +++ b/src/rdb.h @@ -144,7 +144,7 @@ void rdbRemoveTempFile(pid_t childpid); int rdbSave(char *filename, rdbSaveInfo *rsi); ssize_t rdbSaveObject(rio *rdb, robj *o, robj *key); size_t rdbSavedObjectLen(robj *o, robj *key); -robj *rdbLoadObject(int type, rio *rdb, robj *key); +robj *rdbLoadObject(int type, rio *rdb, sds key); void backgroundSaveDoneHandler(int exitcode, int bysignal); int rdbSaveKeyValuePair(rio *rdb, robj *key, robj *val, long long expiretime); ssize_t rdbSaveSingleModuleAux(rio *rdb, int when, moduleType *mt); diff --git a/src/redis-check-rdb.c b/src/redis-check-rdb.c index 1210d49b4..17ec656ce 100644 --- a/src/redis-check-rdb.c +++ b/src/redis-check-rdb.c @@ -287,7 +287,7 @@ int redis_check_rdb(char *rdbfilename, FILE *fp) { rdbstate.keys++; /* Read value */ rdbstate.doing = RDB_CHECK_DOING_READ_OBJECT_VALUE; - if ((val = rdbLoadObject(type,&rdb,key)) == NULL) goto eoferr; + if ((val = rdbLoadObject(type,&rdb,key->ptr)) == NULL) goto eoferr; /* Check if the key already expired. */ if (expiretime != -1 && expiretime < now) rdbstate.already_expired++; diff --git a/src/server.h b/src/server.h index cf4c285f8..b8c46153d 100644 --- a/src/server.h +++ b/src/server.h @@ -2093,8 +2093,8 @@ unsigned int delKeysInSlot(unsigned int hashslot); int verifyClusterConfigWithData(void); void scanGenericCommand(client *c, robj *o, unsigned long cursor); int parseScanCursorOrReply(client *c, robj *o, unsigned long *cursor); -void slotToKeyAdd(robj *key); -void slotToKeyDel(robj *key); +void slotToKeyAdd(sds key); +void slotToKeyDel(sds key); void slotToKeyFlush(void); int dbAsyncDelete(redisDb *db, robj *key); void emptyDbAsync(redisDb *db); From c7e7743d18c9a03fa84f48af0795240af0f4d365 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 9 Apr 2020 11:09:40 +0200 Subject: [PATCH 0259/1280] RDB: clarify a condition in rdbLoadRio(). --- src/rdb.c | 10 ++++++++-- src/rdb.h | 1 + 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/rdb.c b/src/rdb.c index d3104ffbf..313830c25 100644 --- a/src/rdb.c +++ b/src/rdb.c @@ -2231,8 +2231,14 @@ int rdbLoadRio(rio *rdb, int rdbflags, rdbSaveInfo *rsi) { * an RDB file from disk, either at startup, or when an RDB was * received from the master. In the latter case, the master is * responsible for key expiry. If we would expire keys here, the - * snapshot taken by the master may not be reflected on the slave. */ - if (iAmMaster() && !(rdbflags&RDBFLAGS_AOF_PREAMBLE) && expiretime != -1 && expiretime < now) { + * snapshot taken by the master may not be reflected on the slave. + * Similarly if the RDB is the preamble of an AOF file, we want to + * load all the keys as they are, since the log of operations later + * assume to work in an exact keyspace state. */ + if (iAmMaster() && + !(rdbflags&RDBFLAGS_AOF_PREAMBLE) && + expiretime != -1 && expiretime < now) + { sdsfree(key); decrRefCount(val); } else { diff --git a/src/rdb.h b/src/rdb.h index 526ed116b..9dfcae7a5 100644 --- a/src/rdb.h +++ b/src/rdb.h @@ -125,6 +125,7 @@ #define RDBFLAGS_NONE 0 #define RDBFLAGS_AOF_PREAMBLE (1<<0) #define RDBFLAGS_REPLICATION (1<<1) +#define RDBFLAGS_ALLOW_DUP (1<<2) int rdbSaveType(rio *rdb, unsigned char type); int rdbLoadType(rio *rdb); From 911896febb0dd5541c2b044be95347d3c4412547 Mon Sep 17 00:00:00 2001 From: liumiuyong Date: Thu, 9 Apr 2020 17:48:29 +0800 Subject: [PATCH 0260/1280] FIX: truncate max/min longitude,latitude related geo_point (ex: {180, 85.05112878} ) --- src/geohash.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/geohash.c b/src/geohash.c index db5ae025a..de9620b7a 100644 --- a/src/geohash.c +++ b/src/geohash.c @@ -206,7 +206,11 @@ int geohashDecodeWGS84(const GeoHashBits hash, GeoHashArea *area) { int geohashDecodeAreaToLongLat(const GeoHashArea *area, double *xy) { if (!xy) return 0; xy[0] = (area->longitude.min + area->longitude.max) / 2; + if (xy[0] > GEO_LONG_MAX) xy[0] = GEO_LONG_MAX; + if (xy[0] < GEO_LONG_MIN) xy[0] = GEO_LONG_MIN; xy[1] = (area->latitude.min + area->latitude.max) / 2; + if (xy[1] > GEO_LAT_MAX) xy[1] = GEO_LAT_MAX; + if (xy[1] < GEO_LAT_MIN) xy[1] = GEO_LAT_MIN; return 1; } From b10fb451cebbdf3c671125bef5887fb6949ec7de Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 9 Apr 2020 12:02:27 +0200 Subject: [PATCH 0261/1280] More powerful DEBUG RELOAD. Related to #3243. --- src/debug.c | 45 +++++++++++++++++++++++++++++++++++++-------- src/rdb.c | 18 ++++++++++++++---- src/rdb.h | 8 ++++---- 3 files changed, 55 insertions(+), 16 deletions(-) diff --git a/src/debug.c b/src/debug.c index baaaa2424..1351b2536 100644 --- a/src/debug.c +++ b/src/debug.c @@ -366,7 +366,7 @@ void debugCommand(client *c) { "OOM -- Crash the server simulating an out-of-memory error.", "PANIC -- Crash the server simulating a panic.", "POPULATE [prefix] [size] -- Create string keys named key:. If a prefix is specified is used instead of the 'key' prefix.", -"RELOAD -- Save the RDB on disk and reload it back in memory.", +"RELOAD [MERGE] [NOFLUSH] [NOSAVE] -- Save the RDB on disk and reload it back in memory. By default it will save the RDB file and load it back. With the NOFLUSH option the current database is not removed before loading the new one, but conficts in keys will kill the server with an exception. When MERGE is used, conflicting keys will be loaded (the key in the loaded RDB file will win). When NOSAVE is used, the server will not save the current dataset in the RDB file before loading. Use DEBUG RELOAD NOSAVE when you want just to load the RDB file you placed in the Redis working directory in order to replace the current dataset in memory. Use DEBUG RELOAD NOSAVE NOFLUSH MERGE when you want to add what is in the current RDB file placed in the Redis current directory, with the current memory content. Use DEBUG RELOAD when you want to verify Redis is able to persist the current dataset in the RDB file, flush the memory content, and load it back.", "RESTART -- Graceful restart: save config, db, restart.", "SDSLEN -- Show low level SDS string info representing key and value.", "SEGFAULT -- Crash the server with sigsegv.", @@ -411,15 +411,44 @@ NULL serverLog(LL_WARNING, "DEBUG LOG: %s", (char*)c->argv[2]->ptr); addReply(c,shared.ok); } else if (!strcasecmp(c->argv[1]->ptr,"reload")) { - rdbSaveInfo rsi, *rsiptr; - rsiptr = rdbPopulateSaveInfo(&rsi); - if (rdbSave(server.rdb_filename,rsiptr) != C_OK) { - addReply(c,shared.err); - return; + int flush = 1, save = 1; + int flags = RDBFLAGS_NONE; + + /* Parse the additional options that modify the RELOAD + * behavior. */ + for (int j = 2; j < c->argc; j++) { + char *opt = c->argv[j]->ptr; + if (!strcasecmp(opt,"MERGE")) { + flags |= RDBFLAGS_ALLOW_DUP; + } else if (!strcasecmp(opt,"NOFLUSH")) { + flush = 0; + } else if (!strcasecmp(opt,"NOSAVE")) { + save = 0; + } else { + addReplyError(c,"DEBUG RELOAD only supports the " + "MERGE, NOFLUSH and NOSAVE options."); + return; + } } - emptyDb(-1,EMPTYDB_NO_FLAGS,NULL); + + /* The default beahvior is to save the RDB file before loading + * it back. */ + if (save) { + rdbSaveInfo rsi, *rsiptr; + rsiptr = rdbPopulateSaveInfo(&rsi); + if (rdbSave(server.rdb_filename,rsiptr) != C_OK) { + addReply(c,shared.err); + return; + } + } + + /* The default behavior is to remove the current dataset from + * memory before loading the RDB file, however when MERGE is + * used together with NOFLUSH, we are able to merge two datasets. */ + if (flush) emptyDb(-1,EMPTYDB_NO_FLAGS,NULL); + protectClient(c); - int ret = rdbLoad(server.rdb_filename,NULL,RDBFLAGS_NONE); + int ret = rdbLoad(server.rdb_filename,NULL,flags); unprotectClient(c); if (ret != C_OK) { addReplyError(c,"Error trying to load the RDB dump"); diff --git a/src/rdb.c b/src/rdb.c index 313830c25..3f25535ab 100644 --- a/src/rdb.c +++ b/src/rdb.c @@ -2242,18 +2242,28 @@ int rdbLoadRio(rio *rdb, int rdbflags, rdbSaveInfo *rsi) { sdsfree(key); decrRefCount(val); } else { + robj keyobj; + /* Add the new object in the hash table */ int retval = dictAdd(db->dict, key, val); if (retval != DICT_OK) { - serverLog(LL_WARNING, - "RDB has duplicated key '%s' in DB %d",key,db->id); - serverPanic("Duplicated key found in RDB file"); + if (rdbflags & RDBFLAGS_ALLOW_DUP) { + /* This flag is useful for DEBUG RELOAD special modes. + * When it's set we allow new keys to replace the current + * keys with the same name. */ + initStaticStringObject(keyobj,key); + dbSyncDelete(db,&keyobj); + dictAdd(db->dict, key, val); + } else { + serverLog(LL_WARNING, + "RDB has duplicated key '%s' in DB %d",key,db->id); + serverPanic("Duplicated key found in RDB file"); + } } if (server.cluster_enabled) slotToKeyAdd(key); /* Set the expire time if needed */ if (expiretime != -1) { - robj keyobj; initStaticStringObject(keyobj,key); setExpire(NULL,db,&keyobj,expiretime); } diff --git a/src/rdb.h b/src/rdb.h index 9dfcae7a5..aae682dbc 100644 --- a/src/rdb.h +++ b/src/rdb.h @@ -122,10 +122,10 @@ #define RDB_LOAD_SDS (1<<2) /* flags on the purpose of rdb save or load */ -#define RDBFLAGS_NONE 0 -#define RDBFLAGS_AOF_PREAMBLE (1<<0) -#define RDBFLAGS_REPLICATION (1<<1) -#define RDBFLAGS_ALLOW_DUP (1<<2) +#define RDBFLAGS_NONE 0 /* No special RDB loading. */ +#define RDBFLAGS_AOF_PREAMBLE (1<<0) /* Load/save the RDB as AOF preamble. */ +#define RDBFLAGS_REPLICATION (1<<1) /* Load/save for SYNC. */ +#define RDBFLAGS_ALLOW_DUP (1<<2) /* Allow duplicated keys when loading.*/ int rdbSaveType(rio *rdb, unsigned char type); int rdbLoadType(rio *rdb); From df90feb89468a6bb1cc48962920b700b52b928ba Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 9 Apr 2020 16:20:41 +0200 Subject: [PATCH 0262/1280] incrRefCount(): abort on statically allocated object. --- src/object.c | 10 +++++++++- src/server.h | 4 +++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/src/object.c b/src/object.c index 52d5b11f5..1bc400e85 100644 --- a/src/object.c +++ b/src/object.c @@ -347,7 +347,15 @@ void freeStreamObject(robj *o) { } void incrRefCount(robj *o) { - if (o->refcount != OBJ_SHARED_REFCOUNT) o->refcount++; + if (o->refcount < OBJ_FIRST_SPECIAL_REFCOUNT) { + o->refcount++; + } else { + if (o->refcount == OBJ_SHARED_REFCOUNT) { + /* Nothing to do: this refcount is immutable. */ + } else if (o->refcount == OBJ_STATIC_REFCOUNT) { + serverPanic("You tried to retain an object allocated in the stack"); + } + } } void decrRefCount(robj *o) { diff --git a/src/server.h b/src/server.h index b8c46153d..9691381c3 100644 --- a/src/server.h +++ b/src/server.h @@ -597,7 +597,9 @@ typedef struct RedisModuleDigest { #define LRU_CLOCK_MAX ((1<lru */ #define LRU_CLOCK_RESOLUTION 1000 /* LRU clock resolution in ms */ -#define OBJ_SHARED_REFCOUNT INT_MAX +#define OBJ_SHARED_REFCOUNT INT_MAX /* Global object never destroyed. */ +#define OBJ_STATIC_REFCOUNT (INT_MAX-1) /* Object allocated in the stack. */ +#define OBJ_FIRST_SPECIAL_REFCOUNT OBJ_STATIC_REFCOUNT typedef struct redisObject { unsigned type:4; unsigned encoding:4; From 9d24b6f8100d949ae0038f69ecb98cbe92c6d555 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 9 Apr 2020 16:21:48 +0200 Subject: [PATCH 0263/1280] RDB: refactor some RDB loading code into dbAddRDBLoad(). --- src/db.c | 18 ++++++++++++++++++ src/rdb.c | 7 +++---- src/server.h | 1 + 3 files changed, 22 insertions(+), 4 deletions(-) diff --git a/src/db.c b/src/db.c index d393a5fdd..59f0cc7a0 100644 --- a/src/db.c +++ b/src/db.c @@ -188,6 +188,24 @@ void dbAdd(redisDb *db, robj *key, robj *val) { if (server.cluster_enabled) slotToKeyAdd(key->ptr); } +/* This is a special version of dbAdd() that is used only when loading + * keys from the RDB file: the key is passed as an SDS string that is + * retained by the function (and not freed by the caller). + * + * Moreover this function will not abort if the key is already busy, to + * give more control to the caller, nor will signal the key as ready + * since it is not useful in this context. + * + * The function returns 1 if the key was added to the database, taking + * ownership of the SDS string, otherwise 0 is returned, and is up to the + * caller to free the SDS string. */ +int dbAddRDBLoad(redisDb *db, sds key, robj *val) { + int retval = dictAdd(db->dict, key, val); + if (retval != DICT_OK) return 0; + if (server.cluster_enabled) slotToKeyAdd(key); + return 1; +} + /* Overwrite an existing key with a new value. Incrementing the reference * count of the new value is up to the caller. * This function does not modify the expire time of the existing key. diff --git a/src/rdb.c b/src/rdb.c index 3f25535ab..143b6c325 100644 --- a/src/rdb.c +++ b/src/rdb.c @@ -2245,22 +2245,21 @@ int rdbLoadRio(rio *rdb, int rdbflags, rdbSaveInfo *rsi) { robj keyobj; /* Add the new object in the hash table */ - int retval = dictAdd(db->dict, key, val); - if (retval != DICT_OK) { + int added = dbAddRDBLoad(db,key,val); + if (!added) { if (rdbflags & RDBFLAGS_ALLOW_DUP) { /* This flag is useful for DEBUG RELOAD special modes. * When it's set we allow new keys to replace the current * keys with the same name. */ initStaticStringObject(keyobj,key); dbSyncDelete(db,&keyobj); - dictAdd(db->dict, key, val); + dbAddRDBLoad(db,key,val); } else { serverLog(LL_WARNING, "RDB has duplicated key '%s' in DB %d",key,db->id); serverPanic("Duplicated key found in RDB file"); } } - if (server.cluster_enabled) slotToKeyAdd(key); /* Set the expire time if needed */ if (expiretime != -1) { diff --git a/src/server.h b/src/server.h index 9691381c3..c268f2bee 100644 --- a/src/server.h +++ b/src/server.h @@ -2069,6 +2069,7 @@ int objectSetLRUOrLFU(robj *val, long long lfu_freq, long long lru_idle, #define LOOKUP_NONE 0 #define LOOKUP_NOTOUCH (1<<0) void dbAdd(redisDb *db, robj *key, robj *val); +int dbAddRDBLoad(redisDb *db, sds key, robj *val); void dbOverwrite(redisDb *db, robj *key, robj *val); void genericSetKey(redisDb *db, robj *key, robj *val, int keepttl, int signal); void setKey(redisDb *db, robj *key, robj *val); From ce7bfb275d31864582d4d5ac6a8bf16d8afbfcfb Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 9 Apr 2020 16:25:30 +0200 Subject: [PATCH 0264/1280] Use the special static refcount for stack objects. --- src/server.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/server.h b/src/server.h index c268f2bee..d0d5ff154 100644 --- a/src/server.h +++ b/src/server.h @@ -620,7 +620,7 @@ char *getObjectTypeName(robj*); * we'll update it when the structure is changed, to avoid bugs like * bug #85 introduced exactly in this way. */ #define initStaticStringObject(_var,_ptr) do { \ - _var.refcount = 1; \ + _var.refcount = OBJ_STATIC_REFCOUNT; \ _var.type = OBJ_STRING; \ _var.encoding = OBJ_ENCODING_RAW; \ _var.ptr = _ptr; \ From c4dd2bd0cddc4401ea122a21a8d28bba58030cfa Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 10 Apr 2020 10:12:26 +0200 Subject: [PATCH 0265/1280] RESP3: change streams items from maps to arrays. Streams items are similar to dictionaries, however they preserve both the order, and allow for duplicated field names. So a map is not a semantically sounding way to deal with this. https://twitter.com/antirez/status/1248261087553880069 --- src/t_stream.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/t_stream.c b/src/t_stream.c index e0af87f97..4ce3a9b25 100644 --- a/src/t_stream.c +++ b/src/t_stream.c @@ -963,7 +963,7 @@ size_t streamReplyWithRange(client *c, stream *s, streamID *start, streamID *end addReplyArrayLen(c,2); addReplyStreamID(c,&id); - addReplyMapLen(c,numfields); + addReplyArrayLen(c,numfields*2); /* Emit the field-value pairs. */ while(numfields--) { From 517acf75cf81d771bb6a0caf326bf7a2709b9f9e Mon Sep 17 00:00:00 2001 From: Guy Benoish Date: Sat, 11 Apr 2020 15:05:01 +0300 Subject: [PATCH 0266/1280] Typo in getTimeoutFromObjectOrReply's error reply --- src/timeout.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/timeout.c b/src/timeout.c index bb5999418..7787a049f 100644 --- a/src/timeout.c +++ b/src/timeout.c @@ -166,7 +166,7 @@ int getTimeoutFromObjectOrReply(client *c, robj *object, mstime_t *timeout, int if (unit == UNIT_SECONDS) { if (getLongDoubleFromObjectOrReply(c,object,&ftval, - "timeout is not an float or out of range") != C_OK) + "timeout is not a float or out of range") != C_OK) return C_ERR; tval = (long long) (ftval * 1000.0); } else { From 61d84fed333fb1ade3969d6a1c6a295ef4f84c00 Mon Sep 17 00:00:00 2001 From: Jamie Scott Date: Sun, 12 Apr 2020 00:10:19 -0700 Subject: [PATCH 0267/1280] Adding acllog-max-len to Redis.conf While playing with ACLs I noticed that acllog-max-len wasn't in the redis.conf, but was a supported config. This PR documents and adds the directive to the redis.conf file. --- redis.conf | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/redis.conf b/redis.conf index 5baeae65f..fab0a5898 100644 --- a/redis.conf +++ b/redis.conf @@ -737,6 +737,15 @@ replica-priority 100 # For more information about ACL configuration please refer to # the Redis web site at https://redis.io/topics/acl +# ACL LOG +# +# The ACL Log tracks failed commands and authentication events associated +# with ACLs. The ACL Log is useful to troubleshoot failed commands blocked +# by ACLs. The ACL Log is stored in and consumes memory. There is no limit +# to its length.You can reclaim memory with ACL LOG RESET or set a maximum +# length below. +acllog-max-len 128 + # Using an external ACL file # # Instead of configuring users here in this file, it is possible to use From 3ba9724d1637aad9699d652b150ad7860b7016d0 Mon Sep 17 00:00:00 2001 From: Jamie Scott Date: Sun, 12 Apr 2020 17:56:58 -0700 Subject: [PATCH 0268/1280] minor fix --- redis.conf | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/redis.conf b/redis.conf index fab0a5898..01a2d801f 100644 --- a/redis.conf +++ b/redis.conf @@ -741,9 +741,8 @@ replica-priority 100 # # The ACL Log tracks failed commands and authentication events associated # with ACLs. The ACL Log is useful to troubleshoot failed commands blocked -# by ACLs. The ACL Log is stored in and consumes memory. There is no limit -# to its length.You can reclaim memory with ACL LOG RESET or set a maximum -# length below. +# by ACLs. The ACL Log is stored in memory. You can reclaim memory with +# ACL LOG RESET. Define the maximum entry length of the ACL Log below. acllog-max-len 128 # Using an external ACL file From 658f6ea302cae40fd2947b2d98279f1031878aa4 Mon Sep 17 00:00:00 2001 From: Itamar Haber Date: Mon, 13 Apr 2020 17:28:11 +0300 Subject: [PATCH 0269/1280] Prevents default save configuration being reset... ...when using any command line argument --- src/server.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/server.c b/src/server.c index 0afd67514..b07049851 100644 --- a/src/server.c +++ b/src/server.c @@ -5035,7 +5035,6 @@ int main(int argc, char **argv) { "Sentinel needs config file on disk to save state. Exiting..."); exit(1); } - resetServerSaveParams(); loadServerConfig(configfile,options); sdsfree(options); } From 3a31999ecc725f77f0ac606c47df7b0cca24084a Mon Sep 17 00:00:00 2001 From: hwware Date: Tue, 14 Apr 2020 00:16:29 -0400 Subject: [PATCH 0270/1280] fix spelling in acl.c --- src/acl.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/acl.c b/src/acl.c index 733988013..6847130ad 100644 --- a/src/acl.c +++ b/src/acl.c @@ -1783,8 +1783,8 @@ void aclCommand(client *c) { long count = 10; /* Number of entries to emit by default. */ /* Parse the only argument that LOG may have: it could be either - * the number of entires the user wants to display, or alternatively - * the "RESET" command in order to flush the old entires. */ + * the number of entries the user wants to display, or alternatively + * the "RESET" command in order to flush the old entries. */ if (c->argc == 3) { if (!strcasecmp(c->argv[2]->ptr,"reset")) { listSetFreeMethod(ACLLog,ACLFreeLogEntry); From bce3d16a3398ca8daa16414ed3be75c5da45ec01 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 14 Apr 2020 10:52:40 +0200 Subject: [PATCH 0271/1280] Fix function names in zslDeleteNode() top comment. --- src/t_zset.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/t_zset.c b/src/t_zset.c index ea6f4b848..147313d53 100644 --- a/src/t_zset.c +++ b/src/t_zset.c @@ -186,7 +186,8 @@ zskiplistNode *zslInsert(zskiplist *zsl, double score, sds ele) { return x; } -/* Internal function used by zslDelete, zslDeleteByScore and zslDeleteByRank */ +/* Internal function used by zslDelete, zslDeleteRangeByScore and + * zslDeleteRangeByRank. */ void zslDeleteNode(zskiplist *zsl, zskiplistNode *x, zskiplistNode **update) { int i; for (i = 0; i < zsl->level; i++) { From 48bf7424284d7b90e7230307e3dd42cf16fc197c Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 14 Apr 2020 11:23:44 +0200 Subject: [PATCH 0272/1280] Fix zsetAdd() top comment spelling. --- src/t_zset.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/t_zset.c b/src/t_zset.c index 147313d53..5c000e76f 100644 --- a/src/t_zset.c +++ b/src/t_zset.c @@ -1301,14 +1301,14 @@ int zsetScore(robj *zobj, sds member, double *score) { * none could be set if we re-added an element using the same score it used * to have, or in the case a zero increment is used). * - * The function returns 0 on erorr, currently only when the increment + * The function returns 0 on error, currently only when the increment * produces a NAN condition, or when the 'score' value is NAN since the * start. * - * The commad as a side effect of adding a new element may convert the sorted + * The command as a side effect of adding a new element may convert the sorted * set internal encoding from ziplist to hashtable+skiplist. * - * Memory managemnet of 'ele': + * Memory management of 'ele': * * The function does not take ownership of the 'ele' SDS string, but copies * it if needed. */ From 5675053269b0cbc2cf525c99321c96b7c2b39abe Mon Sep 17 00:00:00 2001 From: ShooterIT Date: Tue, 14 Apr 2020 23:56:34 +0800 Subject: [PATCH 0273/1280] Implements sendfile for redis. --- src/config.h | 6 ++++++ src/replication.c | 51 +++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 55 insertions(+), 2 deletions(-) diff --git a/src/config.h b/src/config.h index efa9d11f2..058d73932 100644 --- a/src/config.h +++ b/src/config.h @@ -133,6 +133,12 @@ void setproctitle(const char *fmt, ...); /* Byte ordering detection */ #include /* This will likely define BYTE_ORDER */ +/* Define redis_sendfile. */ +#if defined(__linux__) || (defined(__APPLE__) && defined(MAC_OS_X_VERSION_10_5)) +#define HAVE_SENDFILE 1 +ssize_t redis_sendfile(int out_fd, int in_fd, off_t offset, size_t count); +#endif + #ifndef BYTE_ORDER #if (BSD >= 199103) # include diff --git a/src/replication.c b/src/replication.c index 3e9910374..4a3bbbdcf 100644 --- a/src/replication.c +++ b/src/replication.c @@ -972,10 +972,41 @@ void removeRDBUsedToSyncReplicas(void) { } } +#if HAVE_SENDFILE +/* Implements redis_sendfile to transfer data between file descriptors and + * avoid transferring data to and from user space. + * + * The function prototype is just like sendfile(2) on Linux. in_fd is a file + * descriptor opened for reading and out_fd is a descriptor opened for writing. + * offset specifies where to start reading data from in_fd. count is the number + * of bytes to copy between the file descriptors. + * + * The return value is the number of bytes written to out_fd, if the transfer + * was successful. On error, -1 is returned, and errno is set appropriately. */ +ssize_t redis_sendfile(int out_fd, int in_fd, off_t offset, size_t count) { +#if defined(__linux__) + #include + return sendfile(out_fd, in_fd, &offset, count); + +#elif defined(__APPLE__) + off_t len = count; + /* Notice that it may return -1 and errno is set to EAGAIN even if some + * bytes have been sent successfully and the len argument is set correctly + * when using a socket marked for non-blocking I/O. */ + if (sendfile(in_fd, out_fd, offset, &len, NULL, 0) == -1 && + errno != EAGAIN) return -1; + else + return (ssize_t)len; + +#endif + errno = ENOSYS; + return -1; +} +#endif + void sendBulkToSlave(connection *conn) { client *slave = connGetPrivateData(conn); - char buf[PROTO_IOBUF_LEN]; - ssize_t nwritten, buflen; + ssize_t nwritten; /* Before sending the RDB file, we send the preamble as configured by the * replication process. Currently the preamble is just the bulk count of @@ -1001,6 +1032,21 @@ void sendBulkToSlave(connection *conn) { } /* If the preamble was already transferred, send the RDB bulk data. */ +#if HAVE_SENDFILE + if ((nwritten = redis_sendfile(conn->fd,slave->repldbfd, + slave->repldboff,PROTO_IOBUF_LEN)) == -1) + { + if (errno != EAGAIN) { + serverLog(LL_WARNING,"Sendfile error sending DB to replica: %s", + strerror(errno)); + freeClient(slave); + } + return; + } +#else + ssize_t buflen; + char buf[PROTO_IOBUF_LEN]; + lseek(slave->repldbfd,slave->repldboff,SEEK_SET); buflen = read(slave->repldbfd,buf,PROTO_IOBUF_LEN); if (buflen <= 0) { @@ -1017,6 +1063,7 @@ void sendBulkToSlave(connection *conn) { } return; } +#endif slave->repldboff += nwritten; server.stat_net_output_bytes += nwritten; if (slave->repldboff == slave->repldbsize) { From 7547288a93d74478770598efb413cf82777a4f97 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 15 Apr 2020 15:59:52 +0200 Subject: [PATCH 0274/1280] Fix HELLO reply in Sentinel mode, see #6160. --- src/networking.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/networking.c b/src/networking.c index 654fda517..3dd6bf5da 100644 --- a/src/networking.c +++ b/src/networking.c @@ -2460,7 +2460,7 @@ void helloCommand(client *c) { addReplyBulkCString(c,"mode"); if (server.sentinel_mode) addReplyBulkCString(c,"sentinel"); - if (server.cluster_enabled) addReplyBulkCString(c,"cluster"); + else if (server.cluster_enabled) addReplyBulkCString(c,"cluster"); else addReplyBulkCString(c,"standalone"); if (!server.sentinel_mode) { From 64aac09fc86ff69fecd5b26107e427f3a2126944 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 15 Apr 2020 16:12:06 +0200 Subject: [PATCH 0275/1280] Don't allow empty spaces in ACL key patterns. Fixes issue #6418. --- src/acl.c | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/acl.c b/src/acl.c index 6847130ad..a5e35c4d1 100644 --- a/src/acl.c +++ b/src/acl.c @@ -30,6 +30,7 @@ #include "server.h" #include "sha256.h" #include +#include /* ============================================================================= * Global state for ACLs @@ -690,7 +691,8 @@ void ACLAddAllowedSubcommand(user *u, unsigned long id, const char *sub) { * * When an error is returned, errno is set to the following values: * - * EINVAL: The specified opcode is not understood. + * EINVAL: The specified opcode is not understood or the key pattern is + * invalid (contains non allowed characters). * ENOENT: The command name or command category provided with + or - is not * known. * EBUSY: The subcommand you want to add is about a command that is currently @@ -789,6 +791,15 @@ int ACLSetUser(user *u, const char *op, ssize_t oplen) { errno = EEXIST; return C_ERR; } + /* Validate the pattern: no spaces nor null characters + * are allowed, for simpler rewriting of the ACLs without + * using quoting. */ + for (int i = 1; i < oplen; i++) { + if (isspace(op[i]) || op[i] == 0) { + errno = EINVAL; + return C_ERR; + } + } sds newpat = sdsnewlen(op+1,oplen-1); listNode *ln = listSearchKey(u->patterns,newpat); /* Avoid re-adding the same pattern multiple times. */ From 5e496abef09435790a1873251a37962cbc5d9082 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 15 Apr 2020 16:39:42 +0200 Subject: [PATCH 0276/1280] Don't allow empty spaces in ACL usernames. Fixes issue #6418. --- src/acl.c | 44 ++++++++++++++++++++++++++++++++++++-------- 1 file changed, 36 insertions(+), 8 deletions(-) diff --git a/src/acl.c b/src/acl.c index a5e35c4d1..75b954c5e 100644 --- a/src/acl.c +++ b/src/acl.c @@ -170,6 +170,18 @@ sds ACLHashPassword(unsigned char *cleartext, size_t len) { * Low level ACL API * ==========================================================================*/ +/* Return 1 if the specified string contains spaces or null characters. + * We do this for usernames and key patterns for simpler rewriting of + * ACL rules, presentation on ACL list, and to avoid subtle security bugs + * that may arise from parsing the rules in presence of escapes. + * The function returns 0 if the string has no spaces. */ +int ACLStringHasSpaces(const char *s, size_t len) { + for (size_t i = 0; i < len; i++) { + if (isspace(s[i]) || s[i] == 0) return 1; + } + return 0; +} + /* Given the category name the command returns the corresponding flag, or * zero if there is no match. */ uint64_t ACLGetCommandCategoryFlagByName(const char *name) { @@ -791,14 +803,9 @@ int ACLSetUser(user *u, const char *op, ssize_t oplen) { errno = EEXIST; return C_ERR; } - /* Validate the pattern: no spaces nor null characters - * are allowed, for simpler rewriting of the ACLs without - * using quoting. */ - for (int i = 1; i < oplen; i++) { - if (isspace(op[i]) || op[i] == 0) { - errno = EINVAL; - return C_ERR; - } + if (ACLStringHasSpaces(op+1,oplen-1)) { + errno = EINVAL; + return C_ERR; } sds newpat = sdsnewlen(op+1,oplen-1); listNode *ln = listSearchKey(u->patterns,newpat); @@ -1175,6 +1182,12 @@ int ACLLoadConfiguredUsers(void) { while ((ln = listNext(&li)) != NULL) { sds *aclrules = listNodeValue(ln); sds username = aclrules[0]; + + if (ACLStringHasSpaces(aclrules[0],sdslen(aclrules[0]))) { + serverLog(LL_WARNING,"Spaces not allowed in ACL usernames"); + return C_ERR; + } + user *u = ACLCreateUser(username,sdslen(username)); if (!u) { u = ACLGetUserByName(username,sdslen(username)); @@ -1300,6 +1313,14 @@ sds ACLLoadFromFile(const char *filename) { continue; } + /* Spaces are not allowed in usernames. */ + if (ACLStringHasSpaces(argv[1],sdslen(argv[1]))) { + errors = sdscatprintf(errors, + "'%s:%d: username '%s' contains invalid characters. ", + server.acl_filename, linenum, argv[1]); + continue; + } + /* Try to process the line using the fake user to validate iif * the rules are able to apply cleanly. */ ACLSetUser(fakeuser,"reset",-1); @@ -1609,6 +1630,13 @@ void aclCommand(client *c) { char *sub = c->argv[1]->ptr; if (!strcasecmp(sub,"setuser") && c->argc >= 3) { sds username = c->argv[2]->ptr; + /* Check username validity. */ + if (ACLStringHasSpaces(username,sdslen(username))) { + addReplyErrorFormat(c, + "Usernames can't contain spaces or null characters"); + return; + } + /* Create a temporary user to validate and stage all changes against * before applying to an existing user or creating a new user. If all * arguments are valid the user parameters will all be applied together. From b35324545097b021846223426097626ea3e0f0bc Mon Sep 17 00:00:00 2001 From: hwware Date: Wed, 15 Apr 2020 22:00:36 -0400 Subject: [PATCH 0277/1280] Fix not used marco in cluster.c --- src/cluster.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cluster.c b/src/cluster.c index 2377b386b..eac5ebc54 100644 --- a/src/cluster.c +++ b/src/cluster.c @@ -2106,7 +2106,7 @@ int clusterProcessPacket(clusterLink *link) { resetManualFailover(); server.cluster->mf_end = mstime() + CLUSTER_MF_TIMEOUT; server.cluster->mf_slave = sender; - pauseClients(mstime()+(CLUSTER_MF_TIMEOUT*2)); + pauseClients(mstime()+(CLUSTER_MF_TIMEOUT*CLUSTER_MF_PAUSE_MULT)); serverLog(LL_WARNING,"Manual failover requested by replica %.40s.", sender->name); } else if (type == CLUSTERMSG_TYPE_UPDATE) { From f475453b116c5a917876367c0fc2340a804fe6f9 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 16 Apr 2020 11:21:52 +0200 Subject: [PATCH 0278/1280] RESP3: fix HELLO map len in Sentinel mode. See #6160. --- src/networking.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/networking.c b/src/networking.c index 3dd6bf5da..8f3d79170 100644 --- a/src/networking.c +++ b/src/networking.c @@ -2444,7 +2444,7 @@ void helloCommand(client *c) { /* Let's switch to the specified RESP mode. */ c->resp = ver; - addReplyMapLen(c,7); + addReplyMapLen(c,6 + !server.sentinel_mode); addReplyBulkCString(c,"server"); addReplyBulkCString(c,"redis"); From 7d289b1771254a16f34a34ca33710321048fb8b2 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 16 Apr 2020 16:08:37 +0200 Subject: [PATCH 0279/1280] Update SDS to latest version. --- src/sds.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sds.c b/src/sds.c index 98bd2e77f..118971621 100644 --- a/src/sds.c +++ b/src/sds.c @@ -97,11 +97,11 @@ sds sdsnewlen(const void *init, size_t initlen) { unsigned char *fp; /* flags pointer. */ sh = s_malloc(hdrlen+initlen+1); + if (sh == NULL) return NULL; if (init==SDS_NOINIT) init = NULL; else if (!init) memset(sh, 0, hdrlen+initlen+1); - if (sh == NULL) return NULL; s = (char*)sh+hdrlen; fp = ((unsigned char*)s)-1; switch(type) { From 4ccb0cee4b6acb3917531c9547e3c6ea67b70ba1 Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Thu, 16 Apr 2020 11:05:03 +0300 Subject: [PATCH 0280/1280] testsuite run the defrag latency test solo this test is time sensitive and it sometimes fail to pass below the latency threshold, even on strong machines. this test was the reson we're running just 2 parallel tests in the github actions CI, revering this. --- .github/workflows/ci.yml | 4 ++-- tests/test_helper.tcl | 38 ++++++++++++++++++++++++++++++++++++ tests/unit/memefficiency.tcl | 2 ++ 3 files changed, 42 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3a81d1a08..551fb2d91 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,9 +12,9 @@ jobs: - name: test run: | sudo apt-get install tcl8.5 - ./runtest --clients 2 --verbose + ./runtest --verbose - name: module api test - run: ./runtest-moduleapi --clients 2 --verbose + run: ./runtest-moduleapi --verbose build-ubuntu-old: runs-on: ubuntu-16.04 diff --git a/tests/test_helper.tcl b/tests/test_helper.tcl index d80cb6907..11a804bdc 100644 --- a/tests/test_helper.tcl +++ b/tests/test_helper.tcl @@ -82,6 +82,7 @@ set ::skiptests {} set ::allowtags {} set ::only_tests {} set ::single_tests {} +set ::run_solo_tests {} set ::skip_till "" set ::external 0; # If "1" this means, we are running against external instance set ::file ""; # If set, runs only the tests in this comma separated list @@ -112,6 +113,11 @@ proc execute_tests name { send_data_packet $::test_server_fd done "$name" } +proc execute_code {name code} { + eval $code + send_data_packet $::test_server_fd done "$name" +} + # Setup a list to hold a stack of server configs. When calls to start_server # are nested, use "srv 0 pid" to get the pid of the inner server. To access # outer servers, use "srv -1 pid" etcetera. @@ -188,6 +194,18 @@ proc s {args} { status [srv $level "client"] [lindex $args 0] } +# Test wrapped into run_solo are sent back from the client to the +# test server, so that the test server will send them again to +# clients once the clients are idle. +proc run_solo {name code} { + if {$::numclients == 1 || $::loop || $::external} { + # run_solo is not supported in these scenarios, just run the code. + eval $code + return + } + send_data_packet $::test_server_fd run_solo [list $name $code] +} + proc cleanup {} { if {$::dont_clean} { return @@ -337,6 +355,8 @@ proc read_from_test_client fd { } elseif {$status eq {server-killed}} { set ::active_servers [lsearch -all -inline -not -exact $::active_servers $data] set ::active_clients_task($fd) "(KILLED SERVER) pid:$data" + } elseif {$status eq {run_solo}} { + lappend ::run_solo_tests $data } else { if {!$::quiet} { puts "\[$status\]: $data" @@ -369,6 +389,13 @@ proc force_kill_all_servers {} { } } +proc lpop {listVar {count 1}} { + upvar 1 $listVar l + set ele [lindex $l 0] + set l [lrange $l 1 end] + set ele +} + # A new client is idle. Remove it from the list of active clients and # if there are still test units to run, launch them. proc signal_idle_client fd { @@ -389,6 +416,14 @@ proc signal_idle_client fd { if {$::loop && $::next_test == [llength $::all_tests]} { set ::next_test 0 } + } elseif {[llength $::run_solo_tests] != 0 && [llength $::active_clients] == 0} { + if {!$::quiet} { + puts [colorstr bold-white "Testing solo test"] + set ::active_clients_task($fd) "ASSIGNED: $fd solo test" + } + set ::clients_start_time($fd) [clock seconds] + send_data_packet $fd run_code [lpop ::run_solo_tests] + lappend ::active_clients $fd } else { lappend ::idle_clients $fd set ::active_clients_task($fd) "SLEEPING, no more units to assign" @@ -433,6 +468,9 @@ proc test_client_main server_port { foreach {cmd data} $payload break if {$cmd eq {run}} { execute_tests $data + } elseif {$cmd eq {run_code}} { + foreach {name code} $data break + execute_code $name $code } else { error "Unknown test client command: $cmd" } diff --git a/tests/unit/memefficiency.tcl b/tests/unit/memefficiency.tcl index 06b0e07d7..777693fdf 100644 --- a/tests/unit/memefficiency.tcl +++ b/tests/unit/memefficiency.tcl @@ -36,6 +36,7 @@ start_server {tags {"memefficiency"}} { } } +run_solo {defrag} { start_server {tags {"defrag"}} { if {[string match {*jemalloc*} [s mem_allocator]]} { test "Active defrag" { @@ -328,3 +329,4 @@ start_server {tags {"defrag"}} { } {1} } } +} ;# run_solo From c97372af78583acaa11e31adc30c280d2ec5f134 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 17 Apr 2020 10:51:12 +0200 Subject: [PATCH 0281/1280] A few comments and name changes for #7103. --- tests/test_helper.tcl | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/tests/test_helper.tcl b/tests/test_helper.tcl index 11a804bdc..3eee1aeb7 100644 --- a/tests/test_helper.tcl +++ b/tests/test_helper.tcl @@ -106,14 +106,23 @@ set ::tlsdir "tests/tls" set ::client 0 set ::numclients 16 -proc execute_tests name { +# This function is called by one of the test clients when it receives +# a "run" command from the server, with a filename as data. +# It will run the specified test source file and signal it to the +# test server when finished. +proc execute_test_file name { set path "tests/$name.tcl" set ::curfile $path source $path send_data_packet $::test_server_fd done "$name" } -proc execute_code {name code} { +# This function is called by one of the test clients when it receives +# a "run_code" command from the server, with a verbatim test source code +# as argument, and an associated name. +# It will run the specified code and signal it to the test server when +# finished. +proc execute_test_code {name code} { eval $code send_data_packet $::test_server_fd done "$name" } @@ -467,10 +476,10 @@ proc test_client_main server_port { set payload [read $::test_server_fd $bytes] foreach {cmd data} $payload break if {$cmd eq {run}} { - execute_tests $data + execute_test_file $data } elseif {$cmd eq {run_code}} { foreach {name code} $data break - execute_code $name $code + execute_test_code $name $code } else { error "Unknown test client command: $cmd" } From 901e70ca720576a947a656460f7f579e7028e4a7 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 17 Apr 2020 12:38:12 +0200 Subject: [PATCH 0282/1280] Fix XCLAIM propagation in AOF/replicas for blocking XREADGROUP. See issue #7105. --- src/server.c | 9 +++++++-- src/t_stream.c | 2 +- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/server.c b/src/server.c index 0afd67514..fc9b87aae 100644 --- a/src/server.c +++ b/src/server.c @@ -3082,8 +3082,13 @@ struct redisCommand *lookupCommandOrOriginal(sds name) { * + PROPAGATE_AOF (propagate into the AOF file if is enabled) * + PROPAGATE_REPL (propagate into the replication link) * - * This should not be used inside commands implementation. Use instead - * alsoPropagate(), preventCommandPropagation(), forceCommandPropagation(). + * This should not be used inside commands implementation since it will not + * wrap the resulting commands in MULTI/EXEC. Use instead alsoPropagate(), + * preventCommandPropagation(), forceCommandPropagation(). + * + * However for functions that need to (also) propagate out of the context of a + * command execution, for example when serving a blocked client, you + * want to use propagate(). */ void propagate(struct redisCommand *cmd, int dbid, robj **argv, int argc, int flags) diff --git a/src/t_stream.c b/src/t_stream.c index 4ce3a9b25..155167af9 100644 --- a/src/t_stream.c +++ b/src/t_stream.c @@ -848,7 +848,7 @@ void streamPropagateXCLAIM(client *c, robj *key, streamCG *group, robj *groupnam argv[11] = createStringObject("JUSTID",6); argv[12] = createStringObject("LASTID",6); argv[13] = createObjectFromStreamID(&group->last_id); - alsoPropagate(server.xclaimCommand,c->db->id,argv,14,PROPAGATE_AOF|PROPAGATE_REPL); + propagate(server.xclaimCommand,c->db->id,argv,14,PROPAGATE_AOF|PROPAGATE_REPL); decrRefCount(argv[0]); decrRefCount(argv[3]); decrRefCount(argv[4]); From 189ff2949072373a01035b99f41b4ca47b88d05f Mon Sep 17 00:00:00 2001 From: omg-by <504094596@qq.com> Date: Sat, 18 Apr 2020 00:49:16 +0800 Subject: [PATCH 0283/1280] fix(sentinel): sentinel.running_scripts will always increase more times and not reset when trigger a always fail scripts, sentinel.running_scripts will increase ten times, however it only decrease one times onretry the maximum. and it will't reset, when it become SENTINEL_SCRIPT_MAX_RUNNING, sentinel don't trigger scripts. --- src/sentinel.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/sentinel.c b/src/sentinel.c index 10c003d03..7a25e5ae4 100644 --- a/src/sentinel.c +++ b/src/sentinel.c @@ -859,6 +859,7 @@ void sentinelCollectTerminatedScripts(void) { sj->pid = 0; sj->start_time = mstime() + sentinelScriptRetryDelay(sj->retry_num); + sentinel.running_scripts--; } else { /* Otherwise let's remove the script, but log the event if the * execution did not terminated in the best of the ways. */ From 302db380556fcaf8fbfc8d5ef042ac6e617c9561 Mon Sep 17 00:00:00 2001 From: zhenwei pi Date: Mon, 13 Apr 2020 09:58:35 +0800 Subject: [PATCH 0284/1280] Threaded IO: set thread name for redis-server Set thread name for each thread of redis-server, this helps us to monitor the utilization and optimise the performance. And suggested-by Salvatore, implement this feature for multi platforms. Currently support linux and bsd, ignore other OS. An exmaple on Linux: # top -d 5 -p `pidof redis-server ` -H PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND 3682671 root 20 0 227744 8248 3836 R 99.2 0.0 0:19.53 redis-server 3682677 root 20 0 227744 8248 3836 S 26.4 0.0 0:04.15 io_thd_3 3682675 root 20 0 227744 8248 3836 S 23.6 0.0 0:03.98 io_thd_1 3682676 root 20 0 227744 8248 3836 S 23.6 0.0 0:03.97 io_thd_2 3682672 root 20 0 227744 8248 3836 S 0.2 0.0 0:00.02 bio_close_file 3682673 root 20 0 227744 8248 3836 S 0.2 0.0 0:00.02 bio_aof_fsync 3682674 root 20 0 227744 8248 3836 S 0.0 0.0 0:00.00 bio_lazy_free 3682678 root 20 0 227744 8248 3836 S 0.0 0.0 0:00.00 jemalloc_bg_thd 3682682 root 20 0 227744 8248 3836 S 0.0 0.0 0:00.00 jemalloc_bg_thd 3682683 root 20 0 227744 8248 3836 S 0.0 0.0 0:00.00 jemalloc_bg_thd 3682684 root 20 0 227744 8248 3836 S 0.0 0.0 0:00.00 jemalloc_bg_thd 3682685 root 20 0 227744 8248 3836 S 0.0 0.0 0:00.00 jemalloc_bg_thd 3682687 root 20 0 227744 8248 3836 S 0.0 0.0 0:00.00 jemalloc_bg_thd Another exmaple on FreeBSD-12.1: PID USERNAME PRI NICE SIZE RES STATE C TIME WCPU COMMAND 5212 root 100 0 48M 7280K CPU2 2 0:26 99.52% redis-server{redis-server} 5212 root 38 0 48M 7280K umtxn 4 0:06 26.94% redis-server{io_thd_3} 5212 root 36 0 48M 7280K umtxn 6 0:06 26.84% redis-server{io_thd_1} 5212 root 39 0 48M 7280K umtxn 1 0:06 25.30% redis-server{io_thd_2} 5212 root 20 0 48M 7280K uwait 3 0:00 0.00% redis-server{redis-server} 5212 root 21 0 48M 7280K uwait 2 0:00 0.00% redis-server{bio_close_file} 5212 root 21 0 48M 7280K uwait 3 0:00 0.00% redis-server{bio_aof_fsync} 5212 root 21 0 48M 7280K uwait 0 0:00 0.00% redis-server{bio_lazy_free} Signed-off-by: zhenwei pi --- src/bio.c | 12 ++++++++++++ src/config.h | 12 ++++++++++++ src/networking.c | 4 ++++ 3 files changed, 28 insertions(+) diff --git a/src/bio.c b/src/bio.c index 2af684570..0662c8c4c 100644 --- a/src/bio.c +++ b/src/bio.c @@ -154,6 +154,18 @@ void *bioProcessBackgroundJobs(void *arg) { return NULL; } + switch (type) { + case BIO_CLOSE_FILE: + redis_set_thread_title("bio_close_file"); + break; + case BIO_AOF_FSYNC: + redis_set_thread_title("bio_aof_fsync"); + break; + case BIO_LAZY_FREE: + redis_set_thread_title("bio_lazy_free"); + break; + } + /* Make the thread killable at any time, so that bioKillThreads() * can work reliably. */ pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL); diff --git a/src/config.h b/src/config.h index efa9d11f2..82dc201d3 100644 --- a/src/config.h +++ b/src/config.h @@ -226,4 +226,16 @@ void setproctitle(const char *fmt, ...); #define USE_ALIGNED_ACCESS #endif +/* Define for redis_set_thread_title */ +#ifdef __linux__ +#define redis_set_thread_title(name) pthread_setname_np(pthread_self(), name) +#else +#if (defined __NetBSD__ || defined __FreeBSD__ || defined __OpenBSD__) +#include +#define redis_set_thread_title(name) pthread_set_name_np(pthread_self(), name) +#else +#define redis_set_thread_title(name) +#endif +#endif + #endif diff --git a/src/networking.c b/src/networking.c index 8f3d79170..1f5d0bd5d 100644 --- a/src/networking.c +++ b/src/networking.c @@ -2821,6 +2821,10 @@ void *IOThreadMain(void *myid) { /* The ID is the thread number (from 0 to server.iothreads_num-1), and is * used by the thread to just manipulate a single sub-array of clients. */ long id = (unsigned long)myid; + char thdname[16]; + + snprintf(thdname, sizeof(thdname), "io_thd_%ld", id); + redis_set_thread_title(thdname); while(1) { /* Wait for start */ From b43d2fa48b3ac455f853b8740a0d58e4c2ed2143 Mon Sep 17 00:00:00 2001 From: Guy Benoish Date: Sun, 19 Apr 2020 15:59:58 +0300 Subject: [PATCH 0285/1280] Add the stream tag to XSETID tests --- tests/unit/type/stream.tcl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/type/stream.tcl b/tests/unit/type/stream.tcl index 5de9f0571..c2b524d7f 100644 --- a/tests/unit/type/stream.tcl +++ b/tests/unit/type/stream.tcl @@ -414,7 +414,7 @@ start_server {tags {"stream"} overrides {appendonly yes stream-node-max-entries } } -start_server {tags {"xsetid"}} { +start_server {tags {"stream xsetid"}} { test {XADD can CREATE an empty stream} { r XADD mystream MAXLEN 0 * a b assert {[dict get [r xinfo stream mystream] length] == 0} From 59d1f57172dae4bc717188e9866d1017ff25722f Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 20 Apr 2020 11:52:29 +0200 Subject: [PATCH 0286/1280] Sentinel: small refactoring of sentinelCollectTerminatedScripts(). Related to #7113. --- src/sentinel.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/sentinel.c b/src/sentinel.c index 204a8b317..fb504ae4d 100644 --- a/src/sentinel.c +++ b/src/sentinel.c @@ -860,7 +860,6 @@ void sentinelCollectTerminatedScripts(void) { sj->pid = 0; sj->start_time = mstime() + sentinelScriptRetryDelay(sj->retry_num); - sentinel.running_scripts--; } else { /* Otherwise let's remove the script, but log the event if the * execution did not terminated in the best of the ways. */ @@ -870,8 +869,8 @@ void sentinelCollectTerminatedScripts(void) { } listDelNode(sentinel.scripts_queue,ln); sentinelReleaseScriptJob(sj); - sentinel.running_scripts--; } + sentinel.running_scripts--; } } From 7580f613025800becc6b61cab31393d108551cbd Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 20 Apr 2020 12:17:11 +0200 Subject: [PATCH 0287/1280] Implement redis_set_thread_title for MacOS. Strange enough, pthread_setname_np() produces a warning for not defined function even if pthread is included. Moreover the MacOS documentation claims the return value for the function is void, but actually is int. Related to #7089. --- src/config.h | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/config.h b/src/config.h index 82dc201d3..40dc683ce 100644 --- a/src/config.h +++ b/src/config.h @@ -234,8 +234,14 @@ void setproctitle(const char *fmt, ...); #include #define redis_set_thread_title(name) pthread_set_name_np(pthread_self(), name) #else +#if (defined __APPLE__ && defined(MAC_OS_X_VERSION_10_7)) +int pthread_setname_np(const char *name); +#include +#define redis_set_thread_title(name) pthread_setname_np(name) +#else #define redis_set_thread_title(name) #endif #endif +#endif #endif From 803b0d35e8f00f0547f1458672d4bde1476b9c1f Mon Sep 17 00:00:00 2001 From: Yossi Gottlieb Date: Mon, 20 Apr 2020 13:34:37 +0300 Subject: [PATCH 0288/1280] TLS: Fix build on older verisons of OpenSSL. --- src/tls.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/tls.c b/src/tls.c index 5fac6902b..ea0c34469 100644 --- a/src/tls.c +++ b/src/tls.c @@ -168,7 +168,9 @@ int tlsConfigure(redisTLSContextConfig *ctx_config) { SSL_CTX_set_mode(ctx, SSL_MODE_ENABLE_PARTIAL_WRITE|SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER); SSL_CTX_set_verify(ctx, SSL_VERIFY_PEER|SSL_VERIFY_FAIL_IF_NO_PEER_CERT, NULL); +#if defined(SSL_CTX_set_ecdh_auto) SSL_CTX_set_ecdh_auto(ctx, 1); +#endif if (SSL_CTX_use_certificate_file(ctx, ctx_config->cert_file, SSL_FILETYPE_PEM) <= 0) { ERR_error_string_n(ERR_get_error(), errbuf, sizeof(errbuf)); From fc6e48cdf6d413808fb04e388da1d863a2bcba19 Mon Sep 17 00:00:00 2001 From: Dave-in-lafayette <64047859+Dave-in-lafayette@users.noreply.github.com> Date: Mon, 20 Apr 2020 16:34:36 -0700 Subject: [PATCH 0289/1280] fix for unintended crash during panic response If redis crashes early, before lua is set up (like, if File Descriptor 0 is closed before exec), it will crash again trying to print memory statistics. --- src/server.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/server.c b/src/server.c index fc9b87aae..ebd8c61c1 100644 --- a/src/server.c +++ b/src/server.c @@ -4037,7 +4037,7 @@ sds genRedisInfoString(const char *section) { size_t zmalloc_used = zmalloc_used_memory(); size_t total_system_mem = server.system_memory_size; const char *evict_policy = evictPolicyToString(); - long long memory_lua = (long long)lua_gc(server.lua,LUA_GCCOUNT,0)*1024; + long long memory_lua = server.lua ? (long long)lua_gc(server.lua,LUA_GCCOUNT,0)*1024 : 0; struct redisMemOverhead *mh = getMemoryOverheadData(); /* Peak memory is updated from time to time by serverCron() so it From 986c517239d36de56829bae3085a6ef8755418aa Mon Sep 17 00:00:00 2001 From: Dave-in-lafayette <64047859+Dave-in-lafayette@users.noreply.github.com> Date: Mon, 20 Apr 2020 16:38:06 -0700 Subject: [PATCH 0290/1280] fix for crash during panic before all threads are up If there's a panic before all threads have been started (say, if file descriptor 0 is closed at exec), the panic response will crash here again. --- src/bio.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bio.c b/src/bio.c index 0662c8c4c..85f681185 100644 --- a/src/bio.c +++ b/src/bio.c @@ -266,7 +266,7 @@ void bioKillThreads(void) { int err, j; for (j = 0; j < BIO_NUM_OPS; j++) { - if (pthread_cancel(bio_threads[j]) == 0) { + if (bio_threads[j] && pthread_cancel(bio_threads[j]) == 0) { if ((err = pthread_join(bio_threads[j],NULL)) != 0) { serverLog(LL_WARNING, "Bio thread for job type #%d can be joined: %s", From d3d5108c7d0a9b92e452bd770894a772f5643950 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 21 Apr 2020 10:51:46 +0200 Subject: [PATCH 0291/1280] Tracking: NOLOOP internals implementation. --- src/bitops.c | 8 ++-- src/cluster.c | 4 +- src/db.c | 30 +++++++----- src/debug.c | 2 +- src/expire.c | 6 +-- src/geo.c | 4 +- src/hyperloglog.c | 6 +-- src/module.c | 10 ++-- src/server.h | 12 +++-- src/sort.c | 4 +- src/t_hash.c | 10 ++-- src/t_list.c | 20 ++++---- src/t_set.c | 20 ++++---- src/t_stream.c | 6 +-- src/t_string.c | 14 +++--- src/t_zset.c | 12 ++--- src/tracking.c | 118 ++++++++++++++++++++++++++++++++++------------ 17 files changed, 174 insertions(+), 112 deletions(-) diff --git a/src/bitops.c b/src/bitops.c index 496d78b66..f506a881b 100644 --- a/src/bitops.c +++ b/src/bitops.c @@ -554,7 +554,7 @@ void setbitCommand(client *c) { byteval &= ~(1 << bit); byteval |= ((on & 0x1) << bit); ((uint8_t*)o->ptr)[byte] = byteval; - signalModifiedKey(c->db,c->argv[1]); + signalModifiedKey(c,c->db,c->argv[1]); notifyKeyspaceEvent(NOTIFY_STRING,"setbit",c->argv[1],c->db->id); server.dirty++; addReply(c, bitval ? shared.cone : shared.czero); @@ -754,11 +754,11 @@ void bitopCommand(client *c) { /* Store the computed value into the target key */ if (maxlen) { o = createObject(OBJ_STRING,res); - setKey(c->db,targetkey,o); + setKey(c,c->db,targetkey,o); notifyKeyspaceEvent(NOTIFY_STRING,"set",targetkey,c->db->id); decrRefCount(o); } else if (dbDelete(c->db,targetkey)) { - signalModifiedKey(c->db,targetkey); + signalModifiedKey(c,c->db,targetkey); notifyKeyspaceEvent(NOTIFY_GENERIC,"del",targetkey,c->db->id); } server.dirty++; @@ -1135,7 +1135,7 @@ void bitfieldGeneric(client *c, int flags) { } if (changes) { - signalModifiedKey(c->db,c->argv[1]); + signalModifiedKey(c,c->db,c->argv[1]); notifyKeyspaceEvent(NOTIFY_STRING,"setbit",c->argv[1],c->db->id); server.dirty += changes; } diff --git a/src/cluster.c b/src/cluster.c index 2377b386b..b103e2fe1 100644 --- a/src/cluster.c +++ b/src/cluster.c @@ -4982,7 +4982,7 @@ void restoreCommand(client *c) { setExpire(c,c->db,c->argv[1],ttl); } objectSetLRUOrLFU(obj,lfu_freq,lru_idle,lru_clock,1000); - signalModifiedKey(c->db,c->argv[1]); + signalModifiedKey(c,c->db,c->argv[1]); notifyKeyspaceEvent(NOTIFY_GENERIC,"restore",c->argv[1],c->db->id); addReply(c,shared.ok); server.dirty++; @@ -5329,7 +5329,7 @@ try_again: if (!copy) { /* No COPY option: remove the local key, signal the change. */ dbDelete(c->db,kv[j]); - signalModifiedKey(c->db,kv[j]); + signalModifiedKey(c,c->db,kv[j]); notifyKeyspaceEvent(NOTIFY_GENERIC,"del",kv[j],c->db->id); server.dirty++; diff --git a/src/db.c b/src/db.c index 59f0cc7a0..dc4a0b63e 100644 --- a/src/db.c +++ b/src/db.c @@ -238,8 +238,10 @@ void dbOverwrite(redisDb *db, robj *key, robj *val) { * 3) The expire time of the key is reset (the key is made persistent), * unless 'keepttl' is true. * - * All the new keys in the database should be created via this interface. */ -void genericSetKey(redisDb *db, robj *key, robj *val, int keepttl, int signal) { + * All the new keys in the database should be created via this interface. + * The client 'c' argument may be set to NULL if the operation is performed + * in a context where there is no clear client performing the operation. */ +void genericSetKey(client *c, redisDb *db, robj *key, robj *val, int keepttl, int signal) { if (lookupKeyWrite(db,key) == NULL) { dbAdd(db,key,val); } else { @@ -247,12 +249,12 @@ void genericSetKey(redisDb *db, robj *key, robj *val, int keepttl, int signal) { } incrRefCount(val); if (!keepttl) removeExpire(db,key); - if (signal) signalModifiedKey(db,key); + if (signal) signalModifiedKey(c,db,key); } /* Common case for genericSetKey() where the TTL is not retained. */ -void setKey(redisDb *db, robj *key, robj *val) { - genericSetKey(db,key,val,0,1); +void setKey(client *c, redisDb *db, robj *key, robj *val) { + genericSetKey(c,db,key,val,0,1); } /* Return true if the specified key exists in the specified database. @@ -467,9 +469,11 @@ long long dbTotalServerKeyCount() { * Every time a DB is flushed the function signalFlushDb() is called. *----------------------------------------------------------------------------*/ -void signalModifiedKey(redisDb *db, robj *key) { +/* Note that the 'c' argument may be NULL if the key was modified out of + * a context of a client. */ +void signalModifiedKey(client *c, redisDb *db, robj *key) { touchWatchedKey(db,key); - trackingInvalidateKey(key); + trackingInvalidateKey(c,key); } void signalFlushedDb(int dbid) { @@ -563,7 +567,7 @@ void delGenericCommand(client *c, int lazy) { int deleted = lazy ? dbAsyncDelete(c->db,c->argv[j]) : dbSyncDelete(c->db,c->argv[j]); if (deleted) { - signalModifiedKey(c->db,c->argv[j]); + signalModifiedKey(c,c->db,c->argv[j]); notifyKeyspaceEvent(NOTIFY_GENERIC, "del",c->argv[j],c->db->id); server.dirty++; @@ -1003,8 +1007,8 @@ void renameGenericCommand(client *c, int nx) { dbAdd(c->db,c->argv[2],o); if (expire != -1) setExpire(c,c->db,c->argv[2],expire); dbDelete(c->db,c->argv[1]); - signalModifiedKey(c->db,c->argv[1]); - signalModifiedKey(c->db,c->argv[2]); + signalModifiedKey(c,c->db,c->argv[1]); + signalModifiedKey(c,c->db,c->argv[2]); notifyKeyspaceEvent(NOTIFY_GENERIC,"rename_from", c->argv[1],c->db->id); notifyKeyspaceEvent(NOTIFY_GENERIC,"rename_to", @@ -1072,8 +1076,8 @@ void moveCommand(client *c) { /* OK! key moved, free the entry in the source DB */ dbDelete(src,c->argv[1]); - signalModifiedKey(src,c->argv[1]); - signalModifiedKey(dst,c->argv[1]); + signalModifiedKey(c,src,c->argv[1]); + signalModifiedKey(c,dst,c->argv[1]); notifyKeyspaceEvent(NOTIFY_GENERIC, "move_from",c->argv[1],src->id); notifyKeyspaceEvent(NOTIFY_GENERIC, @@ -1317,7 +1321,7 @@ int expireIfNeeded(redisDb *db, robj *key) { "expired",key,db->id); int retval = server.lazyfree_lazy_expire ? dbAsyncDelete(db,key) : dbSyncDelete(db,key); - if (retval) signalModifiedKey(db,key); + if (retval) signalModifiedKey(NULL,db,key); return retval; } diff --git a/src/debug.c b/src/debug.c index 1351b2536..cbb56cb71 100644 --- a/src/debug.c +++ b/src/debug.c @@ -588,7 +588,7 @@ NULL memcpy(val->ptr, buf, valsize<=buflen? valsize: buflen); } dbAdd(c->db,key,val); - signalModifiedKey(c->db,key); + signalModifiedKey(c,c->db,key); decrRefCount(key); } addReply(c,shared.ok); diff --git a/src/expire.c b/src/expire.c index c102a01ff..30a27193d 100644 --- a/src/expire.c +++ b/src/expire.c @@ -64,7 +64,7 @@ int activeExpireCycleTryExpire(redisDb *db, dictEntry *de, long long now) { dbSyncDelete(db,keyobj); notifyKeyspaceEvent(NOTIFY_EXPIRED, "expired",keyobj,db->id); - trackingInvalidateKey(keyobj); + trackingInvalidateKey(NULL,keyobj); decrRefCount(keyobj); server.stat_expiredkeys++; return 1; @@ -519,14 +519,14 @@ void expireGenericCommand(client *c, long long basetime, int unit) { /* Replicate/AOF this as an explicit DEL or UNLINK. */ aux = server.lazyfree_lazy_expire ? shared.unlink : shared.del; rewriteClientCommandVector(c,2,aux,key); - signalModifiedKey(c->db,key); + signalModifiedKey(c,c->db,key); notifyKeyspaceEvent(NOTIFY_GENERIC,"del",key,c->db->id); addReply(c, shared.cone); return; } else { setExpire(c,c->db,key,when); addReply(c,shared.cone); - signalModifiedKey(c->db,key); + signalModifiedKey(c,c->db,key); notifyKeyspaceEvent(NOTIFY_GENERIC,"expire",key,c->db->id); server.dirty++; return; diff --git a/src/geo.c b/src/geo.c index f7920a2e2..3e5d5f606 100644 --- a/src/geo.c +++ b/src/geo.c @@ -657,13 +657,13 @@ void georadiusGeneric(client *c, int flags) { if (returned_items) { zsetConvertToZiplistIfNeeded(zobj,maxelelen); - setKey(c->db,storekey,zobj); + setKey(c,c->db,storekey,zobj); decrRefCount(zobj); notifyKeyspaceEvent(NOTIFY_ZSET,"georadiusstore",storekey, c->db->id); server.dirty += returned_items; } else if (dbDelete(c->db,storekey)) { - signalModifiedKey(c->db,storekey); + signalModifiedKey(c,c->db,storekey); notifyKeyspaceEvent(NOTIFY_GENERIC,"del",storekey,c->db->id); server.dirty++; } diff --git a/src/hyperloglog.c b/src/hyperloglog.c index facd99743..721f492a1 100644 --- a/src/hyperloglog.c +++ b/src/hyperloglog.c @@ -1209,7 +1209,7 @@ void pfaddCommand(client *c) { } hdr = o->ptr; if (updated) { - signalModifiedKey(c->db,c->argv[1]); + signalModifiedKey(c,c->db,c->argv[1]); notifyKeyspaceEvent(NOTIFY_STRING,"pfadd",c->argv[1],c->db->id); server.dirty++; HLL_INVALIDATE_CACHE(hdr); @@ -1300,7 +1300,7 @@ void pfcountCommand(client *c) { * data structure is not modified, since the cached value * may be modified and given that the HLL is a Redis string * we need to propagate the change. */ - signalModifiedKey(c->db,c->argv[1]); + signalModifiedKey(c,c->db,c->argv[1]); server.dirty++; } addReplyLongLong(c,card); @@ -1373,7 +1373,7 @@ void pfmergeCommand(client *c) { last hllSparseSet() call. */ HLL_INVALIDATE_CACHE(hdr); - signalModifiedKey(c->db,c->argv[1]); + signalModifiedKey(c,c->db,c->argv[1]); /* We generate a PFADD event for PFMERGE for semantical simplicity * since in theory this is a mass-add of elements. */ notifyKeyspaceEvent(NOTIFY_STRING,"pfadd",c->argv[1],c->db->id); diff --git a/src/module.c b/src/module.c index 4d3d9e1af..e3a338dad 100644 --- a/src/module.c +++ b/src/module.c @@ -896,7 +896,7 @@ void RM_SetModuleOptions(RedisModuleCtx *ctx, int options) { /* Signals that the key is modified from user's perspective (i.e. invalidate WATCH * and client side caching). */ int RM_SignalModifiedKey(RedisModuleCtx *ctx, RedisModuleString *keyname) { - signalModifiedKey(ctx->client->db,keyname); + signalModifiedKey(ctx->client,ctx->client->db,keyname); return REDISMODULE_OK; } @@ -2016,7 +2016,7 @@ void *RM_OpenKey(RedisModuleCtx *ctx, robj *keyname, int mode) { static void moduleCloseKey(RedisModuleKey *key) { int signal = SHOULD_SIGNAL_MODIFIED_KEYS(key->ctx); if ((key->mode & REDISMODULE_WRITE) && signal) - signalModifiedKey(key->db,key->key); + signalModifiedKey(key->ctx->client,key->db,key->key); /* TODO: if (key->iter) RM_KeyIteratorStop(kp); */ RM_ZsetRangeStop(key); decrRefCount(key->key); @@ -2157,7 +2157,7 @@ RedisModuleString *RM_RandomKey(RedisModuleCtx *ctx) { int RM_StringSet(RedisModuleKey *key, RedisModuleString *str) { if (!(key->mode & REDISMODULE_WRITE) || key->iter) return REDISMODULE_ERR; RM_DeleteKey(key); - genericSetKey(key->db,key->key,str,0,0); + genericSetKey(key->ctx->client,key->db,key->key,str,0,0); key->value = str; return REDISMODULE_OK; } @@ -2237,7 +2237,7 @@ int RM_StringTruncate(RedisModuleKey *key, size_t newlen) { if (key->value == NULL) { /* Empty key: create it with the new size. */ robj *o = createObject(OBJ_STRING,sdsnewlen(NULL, newlen)); - genericSetKey(key->db,key->key,o,0,0); + genericSetKey(key->ctx->client,key->db,key->key,o,0,0); key->value = o; decrRefCount(o); } else { @@ -3625,7 +3625,7 @@ int RM_ModuleTypeSetValue(RedisModuleKey *key, moduleType *mt, void *value) { if (!(key->mode & REDISMODULE_WRITE) || key->iter) return REDISMODULE_ERR; RM_DeleteKey(key); robj *o = createModuleObject(mt,value); - genericSetKey(key->db,key->key,o,0,0); + genericSetKey(key->ctx->client,key->db,key->key,o,0,0); decrRefCount(o); key->value = o; return REDISMODULE_OK; diff --git a/src/server.h b/src/server.h index d0d5ff154..d39359dce 100644 --- a/src/server.h +++ b/src/server.h @@ -252,7 +252,9 @@ typedef long long ustime_t; /* microsecond time type. */ #define CLIENT_TRACKING_OPTOUT (1ULL<<35) /* Tracking in opt-out mode. */ #define CLIENT_TRACKING_CACHING (1ULL<<36) /* CACHING yes/no was given, depending on optin/optout mode. */ -#define CLIENT_IN_TO_TABLE (1ULL<<37) /* This client is in the timeout table. */ +#define CLIENT_TRACKING_NOLOOP (1ULL<<37) /* Don't send invalidation messages + about writes performed by myself.*/ +#define CLIENT_IN_TO_TABLE (1ULL<<38) /* This client is in the timeout table. */ /* Client block type (btype field in client structure) * if CLIENT_BLOCKED flag is set. */ @@ -1683,7 +1685,7 @@ void addReplyStatusFormat(client *c, const char *fmt, ...); void enableTracking(client *c, uint64_t redirect_to, uint64_t options, robj **prefix, size_t numprefix); void disableTracking(client *c); void trackingRememberKeys(client *c); -void trackingInvalidateKey(robj *keyobj); +void trackingInvalidateKey(client *c, robj *keyobj); void trackingInvalidateKeysOnFlush(int dbid); void trackingLimitUsedSlots(void); uint64_t trackingGetTotalItems(void); @@ -2071,8 +2073,8 @@ int objectSetLRUOrLFU(robj *val, long long lfu_freq, long long lru_idle, void dbAdd(redisDb *db, robj *key, robj *val); int dbAddRDBLoad(redisDb *db, sds key, robj *val); void dbOverwrite(redisDb *db, robj *key, robj *val); -void genericSetKey(redisDb *db, robj *key, robj *val, int keepttl, int signal); -void setKey(redisDb *db, robj *key, robj *val); +void genericSetKey(client *c, redisDb *db, robj *key, robj *val, int keepttl, int signal); +void setKey(client *c, redisDb *db, robj *key, robj *val); int dbExists(redisDb *db, robj *key); robj *dbRandomKey(redisDb *db); int dbSyncDelete(redisDb *db, robj *key); @@ -2088,7 +2090,7 @@ void flushAllDataAndResetRDB(int flags); long long dbTotalServerKeyCount(); int selectDb(client *c, int id); -void signalModifiedKey(redisDb *db, robj *key); +void signalModifiedKey(client *c, redisDb *db, robj *key); void signalFlushedDb(int dbid); unsigned int getKeysInSlot(unsigned int hashslot, robj **keys, unsigned int count); unsigned int countKeysInSlot(unsigned int hashslot); diff --git a/src/sort.c b/src/sort.c index db26da158..f269a7731 100644 --- a/src/sort.c +++ b/src/sort.c @@ -570,12 +570,12 @@ void sortCommand(client *c) { } } if (outputlen) { - setKey(c->db,storekey,sobj); + setKey(c,c->db,storekey,sobj); notifyKeyspaceEvent(NOTIFY_LIST,"sortstore",storekey, c->db->id); server.dirty += outputlen; } else if (dbDelete(c->db,storekey)) { - signalModifiedKey(c->db,storekey); + signalModifiedKey(c,c->db,storekey); notifyKeyspaceEvent(NOTIFY_GENERIC,"del",storekey,c->db->id); server.dirty++; } diff --git a/src/t_hash.c b/src/t_hash.c index b9f0db7fc..866bcd25b 100644 --- a/src/t_hash.c +++ b/src/t_hash.c @@ -521,7 +521,7 @@ void hsetnxCommand(client *c) { } else { hashTypeSet(o,c->argv[2]->ptr,c->argv[3]->ptr,HASH_SET_COPY); addReply(c, shared.cone); - signalModifiedKey(c->db,c->argv[1]); + signalModifiedKey(c,c->db,c->argv[1]); notifyKeyspaceEvent(NOTIFY_HASH,"hset",c->argv[1],c->db->id); server.dirty++; } @@ -551,7 +551,7 @@ void hsetCommand(client *c) { /* HMSET */ addReply(c, shared.ok); } - signalModifiedKey(c->db,c->argv[1]); + signalModifiedKey(c,c->db,c->argv[1]); notifyKeyspaceEvent(NOTIFY_HASH,"hset",c->argv[1],c->db->id); server.dirty++; } @@ -586,7 +586,7 @@ void hincrbyCommand(client *c) { new = sdsfromlonglong(value); hashTypeSet(o,c->argv[2]->ptr,new,HASH_SET_TAKE_VALUE); addReplyLongLong(c,value); - signalModifiedKey(c->db,c->argv[1]); + signalModifiedKey(c,c->db,c->argv[1]); notifyKeyspaceEvent(NOTIFY_HASH,"hincrby",c->argv[1],c->db->id); server.dirty++; } @@ -625,7 +625,7 @@ void hincrbyfloatCommand(client *c) { new = sdsnewlen(buf,len); hashTypeSet(o,c->argv[2]->ptr,new,HASH_SET_TAKE_VALUE); addReplyBulkCBuffer(c,buf,len); - signalModifiedKey(c->db,c->argv[1]); + signalModifiedKey(c,c->db,c->argv[1]); notifyKeyspaceEvent(NOTIFY_HASH,"hincrbyfloat",c->argv[1],c->db->id); server.dirty++; @@ -721,7 +721,7 @@ void hdelCommand(client *c) { } } if (deleted) { - signalModifiedKey(c->db,c->argv[1]); + signalModifiedKey(c,c->db,c->argv[1]); notifyKeyspaceEvent(NOTIFY_HASH,"hdel",c->argv[1],c->db->id); if (keyremoved) notifyKeyspaceEvent(NOTIFY_GENERIC,"del",c->argv[1], diff --git a/src/t_list.c b/src/t_list.c index eaeaa8e48..4770a2272 100644 --- a/src/t_list.c +++ b/src/t_list.c @@ -217,7 +217,7 @@ void pushGenericCommand(client *c, int where) { if (pushed) { char *event = (where == LIST_HEAD) ? "lpush" : "rpush"; - signalModifiedKey(c->db,c->argv[1]); + signalModifiedKey(c,c->db,c->argv[1]); notifyKeyspaceEvent(NOTIFY_LIST,event,c->argv[1],c->db->id); } server.dirty += pushed; @@ -247,7 +247,7 @@ void pushxGenericCommand(client *c, int where) { if (pushed) { char *event = (where == LIST_HEAD) ? "lpush" : "rpush"; - signalModifiedKey(c->db,c->argv[1]); + signalModifiedKey(c,c->db,c->argv[1]); notifyKeyspaceEvent(NOTIFY_LIST,event,c->argv[1],c->db->id); } server.dirty += pushed; @@ -292,7 +292,7 @@ void linsertCommand(client *c) { listTypeReleaseIterator(iter); if (inserted) { - signalModifiedKey(c->db,c->argv[1]); + signalModifiedKey(c,c->db,c->argv[1]); notifyKeyspaceEvent(NOTIFY_LIST,"linsert", c->argv[1],c->db->id); server.dirty++; @@ -355,7 +355,7 @@ void lsetCommand(client *c) { addReply(c,shared.outofrangeerr); } else { addReply(c,shared.ok); - signalModifiedKey(c->db,c->argv[1]); + signalModifiedKey(c,c->db,c->argv[1]); notifyKeyspaceEvent(NOTIFY_LIST,"lset",c->argv[1],c->db->id); server.dirty++; } @@ -382,7 +382,7 @@ void popGenericCommand(client *c, int where) { c->argv[1],c->db->id); dbDelete(c->db,c->argv[1]); } - signalModifiedKey(c->db,c->argv[1]); + signalModifiedKey(c,c->db,c->argv[1]); server.dirty++; } } @@ -482,7 +482,7 @@ void ltrimCommand(client *c) { dbDelete(c->db,c->argv[1]); notifyKeyspaceEvent(NOTIFY_GENERIC,"del",c->argv[1],c->db->id); } - signalModifiedKey(c->db,c->argv[1]); + signalModifiedKey(c,c->db,c->argv[1]); server.dirty++; addReply(c,shared.ok); } @@ -519,7 +519,7 @@ void lremCommand(client *c) { listTypeReleaseIterator(li); if (removed) { - signalModifiedKey(c->db,c->argv[1]); + signalModifiedKey(c,c->db,c->argv[1]); notifyKeyspaceEvent(NOTIFY_LIST,"lrem",c->argv[1],c->db->id); } @@ -555,7 +555,7 @@ void rpoplpushHandlePush(client *c, robj *dstkey, robj *dstobj, robj *value) { server.list_compress_depth); dbAdd(c->db,dstkey,dstobj); } - signalModifiedKey(c->db,dstkey); + signalModifiedKey(c,c->db,dstkey); listTypePush(dstobj,value,LIST_HEAD); notifyKeyspaceEvent(NOTIFY_LIST,"lpush",dstkey,c->db->id); /* Always send the pushed value to the client. */ @@ -593,7 +593,7 @@ void rpoplpushCommand(client *c) { notifyKeyspaceEvent(NOTIFY_GENERIC,"del", touchedkey,c->db->id); } - signalModifiedKey(c->db,touchedkey); + signalModifiedKey(c,c->db,touchedkey); decrRefCount(touchedkey); server.dirty++; if (c->cmd->proc == brpoplpushCommand) { @@ -708,7 +708,7 @@ void blockingPopGenericCommand(client *c, int where) { notifyKeyspaceEvent(NOTIFY_GENERIC,"del", c->argv[j],c->db->id); } - signalModifiedKey(c->db,c->argv[j]); + signalModifiedKey(c,c->db,c->argv[j]); server.dirty++; /* Replicate it as an [LR]POP instead of B[LR]POP. */ diff --git a/src/t_set.c b/src/t_set.c index 60cf22d8c..c2e73a6e6 100644 --- a/src/t_set.c +++ b/src/t_set.c @@ -280,7 +280,7 @@ void saddCommand(client *c) { if (setTypeAdd(set,c->argv[j]->ptr)) added++; } if (added) { - signalModifiedKey(c->db,c->argv[1]); + signalModifiedKey(c,c->db,c->argv[1]); notifyKeyspaceEvent(NOTIFY_SET,"sadd",c->argv[1],c->db->id); } server.dirty += added; @@ -305,7 +305,7 @@ void sremCommand(client *c) { } } if (deleted) { - signalModifiedKey(c->db,c->argv[1]); + signalModifiedKey(c,c->db,c->argv[1]); notifyKeyspaceEvent(NOTIFY_SET,"srem",c->argv[1],c->db->id); if (keyremoved) notifyKeyspaceEvent(NOTIFY_GENERIC,"del",c->argv[1], @@ -358,8 +358,8 @@ void smoveCommand(client *c) { dbAdd(c->db,c->argv[2],dstset); } - signalModifiedKey(c->db,c->argv[1]); - signalModifiedKey(c->db,c->argv[2]); + signalModifiedKey(c,c->db,c->argv[1]); + signalModifiedKey(c,c->db,c->argv[2]); server.dirty++; /* An extra key has changed when ele was successfully added to dstset */ @@ -444,7 +444,7 @@ void spopWithCountCommand(client *c) { /* Propagate this command as an DEL operation */ rewriteClientCommandVector(c,2,shared.del,c->argv[1]); - signalModifiedKey(c->db,c->argv[1]); + signalModifiedKey(c,c->db,c->argv[1]); server.dirty++; return; } @@ -546,7 +546,7 @@ void spopWithCountCommand(client *c) { * the alsoPropagate() API. */ decrRefCount(propargv[0]); preventCommandPropagation(c); - signalModifiedKey(c->db,c->argv[1]); + signalModifiedKey(c,c->db,c->argv[1]); server.dirty++; } @@ -599,7 +599,7 @@ void spopCommand(client *c) { } /* Set has been modified */ - signalModifiedKey(c->db,c->argv[1]); + signalModifiedKey(c,c->db,c->argv[1]); server.dirty++; } @@ -808,7 +808,7 @@ void sinterGenericCommand(client *c, robj **setkeys, zfree(sets); if (dstkey) { if (dbDelete(c->db,dstkey)) { - signalModifiedKey(c->db,dstkey); + signalModifiedKey(c,c->db,dstkey); server.dirty++; } addReply(c,shared.czero); @@ -908,7 +908,7 @@ void sinterGenericCommand(client *c, robj **setkeys, notifyKeyspaceEvent(NOTIFY_GENERIC,"del", dstkey,c->db->id); } - signalModifiedKey(c->db,dstkey); + signalModifiedKey(c,c->db,dstkey); server.dirty++; } else { setDeferredSetLen(c,replylen,cardinality); @@ -1083,7 +1083,7 @@ void sunionDiffGenericCommand(client *c, robj **setkeys, int setnum, notifyKeyspaceEvent(NOTIFY_GENERIC,"del", dstkey,c->db->id); } - signalModifiedKey(c->db,dstkey); + signalModifiedKey(c,c->db,dstkey); server.dirty++; } zfree(sets); diff --git a/src/t_stream.c b/src/t_stream.c index 155167af9..3efaa4509 100644 --- a/src/t_stream.c +++ b/src/t_stream.c @@ -1262,7 +1262,7 @@ void xaddCommand(client *c) { } addReplyStreamID(c,&id); - signalModifiedKey(c->db,c->argv[1]); + signalModifiedKey(c,c->db,c->argv[1]); notifyKeyspaceEvent(NOTIFY_STREAM,"xadd",c->argv[1],c->db->id); server.dirty++; @@ -2390,7 +2390,7 @@ void xdelCommand(client *c) { /* Propagate the write if needed. */ if (deleted) { - signalModifiedKey(c->db,c->argv[1]); + signalModifiedKey(c,c->db,c->argv[1]); notifyKeyspaceEvent(NOTIFY_STREAM,"xdel",c->argv[1],c->db->id); server.dirty += deleted; } @@ -2467,7 +2467,7 @@ void xtrimCommand(client *c) { /* Propagate the write if needed. */ if (deleted) { - signalModifiedKey(c->db,c->argv[1]); + signalModifiedKey(c,c->db,c->argv[1]); notifyKeyspaceEvent(NOTIFY_STREAM,"xtrim",c->argv[1],c->db->id); server.dirty += deleted; if (approx_maxlen) streamRewriteApproxMaxlen(c,s,maxlen_arg_idx); diff --git a/src/t_string.c b/src/t_string.c index 335bda404..ef382bb0c 100644 --- a/src/t_string.c +++ b/src/t_string.c @@ -84,7 +84,7 @@ void setGenericCommand(client *c, int flags, robj *key, robj *val, robj *expire, addReply(c, abort_reply ? abort_reply : shared.null[c->resp]); return; } - genericSetKey(c->db,key,val,flags & OBJ_SET_KEEPTTL,1); + genericSetKey(c,c->db,key,val,flags & OBJ_SET_KEEPTTL,1); server.dirty++; if (expire) setExpire(c,c->db,key,mstime()+milliseconds); notifyKeyspaceEvent(NOTIFY_STRING,"set",key,c->db->id); @@ -183,7 +183,7 @@ void getCommand(client *c) { void getsetCommand(client *c) { if (getGenericCommand(c) == C_ERR) return; c->argv[2] = tryObjectEncoding(c->argv[2]); - setKey(c->db,c->argv[1],c->argv[2]); + setKey(c,c->db,c->argv[1],c->argv[2]); notifyKeyspaceEvent(NOTIFY_STRING,"set",c->argv[1],c->db->id); server.dirty++; } @@ -240,7 +240,7 @@ void setrangeCommand(client *c) { if (sdslen(value) > 0) { o->ptr = sdsgrowzero(o->ptr,offset+sdslen(value)); memcpy((char*)o->ptr+offset,value,sdslen(value)); - signalModifiedKey(c->db,c->argv[1]); + signalModifiedKey(c,c->db,c->argv[1]); notifyKeyspaceEvent(NOTIFY_STRING, "setrange",c->argv[1],c->db->id); server.dirty++; @@ -328,7 +328,7 @@ void msetGenericCommand(client *c, int nx) { for (j = 1; j < c->argc; j += 2) { c->argv[j+1] = tryObjectEncoding(c->argv[j+1]); - setKey(c->db,c->argv[j],c->argv[j+1]); + setKey(c,c->db,c->argv[j],c->argv[j+1]); notifyKeyspaceEvent(NOTIFY_STRING,"set",c->argv[j],c->db->id); } server.dirty += (c->argc-1)/2; @@ -373,7 +373,7 @@ void incrDecrCommand(client *c, long long incr) { dbAdd(c->db,c->argv[1],new); } } - signalModifiedKey(c->db,c->argv[1]); + signalModifiedKey(c,c->db,c->argv[1]); notifyKeyspaceEvent(NOTIFY_STRING,"incrby",c->argv[1],c->db->id); server.dirty++; addReply(c,shared.colon); @@ -423,7 +423,7 @@ void incrbyfloatCommand(client *c) { dbOverwrite(c->db,c->argv[1],new); else dbAdd(c->db,c->argv[1],new); - signalModifiedKey(c->db,c->argv[1]); + signalModifiedKey(c,c->db,c->argv[1]); notifyKeyspaceEvent(NOTIFY_STRING,"incrbyfloat",c->argv[1],c->db->id); server.dirty++; addReplyBulk(c,new); @@ -467,7 +467,7 @@ void appendCommand(client *c) { o->ptr = sdscatlen(o->ptr,append->ptr,sdslen(append->ptr)); totlen = sdslen(o->ptr); } - signalModifiedKey(c->db,c->argv[1]); + signalModifiedKey(c,c->db,c->argv[1]); notifyKeyspaceEvent(NOTIFY_STRING,"append",c->argv[1],c->db->id); server.dirty++; addReplyLongLong(c,totlen); diff --git a/src/t_zset.c b/src/t_zset.c index 5c000e76f..9c409cd96 100644 --- a/src/t_zset.c +++ b/src/t_zset.c @@ -1646,7 +1646,7 @@ reply_to_client: cleanup: zfree(scores); if (added || updated) { - signalModifiedKey(c->db,key); + signalModifiedKey(c,c->db,key); notifyKeyspaceEvent(NOTIFY_ZSET, incr ? "zincr" : "zadd", key, c->db->id); } @@ -1681,7 +1681,7 @@ void zremCommand(client *c) { notifyKeyspaceEvent(NOTIFY_ZSET,"zrem",key,c->db->id); if (keyremoved) notifyKeyspaceEvent(NOTIFY_GENERIC,"del",key,c->db->id); - signalModifiedKey(c->db,key); + signalModifiedKey(c,c->db,key); server.dirty += deleted; } addReplyLongLong(c,deleted); @@ -1779,7 +1779,7 @@ void zremrangeGenericCommand(client *c, int rangetype) { /* Step 4: Notifications and reply. */ if (deleted) { char *event[3] = {"zremrangebyrank","zremrangebyscore","zremrangebylex"}; - signalModifiedKey(c->db,key); + signalModifiedKey(c,c->db,key); notifyKeyspaceEvent(NOTIFY_ZSET,event[rangetype],key,c->db->id); if (keyremoved) notifyKeyspaceEvent(NOTIFY_GENERIC,"del",key,c->db->id); @@ -2383,7 +2383,7 @@ void zunionInterGenericCommand(client *c, robj *dstkey, int op) { zsetConvertToZiplistIfNeeded(dstobj,maxelelen); dbAdd(c->db,dstkey,dstobj); addReplyLongLong(c,zsetLength(dstobj)); - signalModifiedKey(c->db,dstkey); + signalModifiedKey(c,c->db,dstkey); notifyKeyspaceEvent(NOTIFY_ZSET, (op == SET_OP_UNION) ? "zunionstore" : "zinterstore", dstkey,c->db->id); @@ -2392,7 +2392,7 @@ void zunionInterGenericCommand(client *c, robj *dstkey, int op) { decrRefCount(dstobj); addReply(c,shared.czero); if (touched) { - signalModifiedKey(c->db,dstkey); + signalModifiedKey(c,c->db,dstkey); notifyKeyspaceEvent(NOTIFY_GENERIC,"del",dstkey,c->db->id); server.dirty++; } @@ -3216,7 +3216,7 @@ void genericZpopCommand(client *c, robj **keyv, int keyc, int where, int emitkey if (arraylen == 0) { /* Do this only for the first iteration. */ char *events[2] = {"zpopmin","zpopmax"}; notifyKeyspaceEvent(NOTIFY_ZSET,events[where],key,c->db->id); - signalModifiedKey(c->db,key); + signalModifiedKey(c,c->db,key); } addReplyBulkCBuffer(c,ele,sdslen(ele)); diff --git a/src/tracking.c b/src/tracking.c index 6f7929430..434e086b5 100644 --- a/src/tracking.c +++ b/src/tracking.c @@ -245,7 +245,7 @@ void sendTrackingMessage(client *c, char *keyname, size_t keylen, int proto) { * matches one or more prefixes in the prefix table. Later when we * return to the event loop, we'll send invalidation messages to the * clients subscribed to each prefix. */ -void trackingRememberKeyToBroadcast(char *keyname, size_t keylen) { +void trackingRememberKeyToBroadcast(client *c, char *keyname, size_t keylen) { raxIterator ri; raxStart(&ri,PrefixTable); raxSeek(&ri,"^",NULL,0); @@ -254,7 +254,11 @@ void trackingRememberKeyToBroadcast(char *keyname, size_t keylen) { if (ri.key_len != 0 && memcmp(ri.key,keyname,ri.key_len) != 0) continue; bcastState *bs = ri.data; - raxTryInsert(bs->keys,(unsigned char*)keyname,keylen,NULL,NULL); + /* We insert the client pointer as associated value in the radix + * tree. This way we know who was the client that did the last + * change to the key, and can avoid sending the notification in the + * case the client is in NOLOOP mode. */ + raxTryInsert(bs->keys,(unsigned char*)keyname,keylen,c,NULL); } raxStop(&ri); } @@ -262,13 +266,17 @@ void trackingRememberKeyToBroadcast(char *keyname, size_t keylen) { /* This function is called from signalModifiedKey() or other places in Redis * when a key changes value. In the context of keys tracking, our task here is * to send a notification to every client that may have keys about such caching - * slot. */ -void trackingInvalidateKey(robj *keyobj) { + * slot. + * + * Note that 'c' may be NULL in case the operation was performed outside the + * context of a client modifying the database (for instance when we delete a + * key because of expire). */ +void trackingInvalidateKey(client *c, robj *keyobj) { if (TrackingTable == NULL) return; sds sdskey = keyobj->ptr; if (raxSize(PrefixTable) > 0) - trackingRememberKeyToBroadcast(sdskey,sdslen(sdskey)); + trackingRememberKeyToBroadcast(c,sdskey,sdslen(sdskey)); rax *ids = raxFind(TrackingTable,(unsigned char*)sdskey,sdslen(sdskey)); if (ids == raxNotFound) return; @@ -279,19 +287,28 @@ void trackingInvalidateKey(robj *keyobj) { while(raxNext(&ri)) { uint64_t id; memcpy(&id,ri.key,sizeof(id)); - client *c = lookupClientByID(id); + client *target = lookupClientByID(id); /* Note that if the client is in BCAST mode, we don't want to * send invalidation messages that were pending in the case * previously the client was not in BCAST mode. This can happen if * TRACKING is enabled normally, and then the client switches to * BCAST mode. */ - if (c == NULL || - !(c->flags & CLIENT_TRACKING)|| - c->flags & CLIENT_TRACKING_BCAST) + if (target == NULL || + !(target->flags & CLIENT_TRACKING)|| + target->flags & CLIENT_TRACKING_BCAST) { continue; } - sendTrackingMessage(c,sdskey,sdslen(sdskey),0); + + /* If the client enabled the NOLOOP mode, don't send notifications + * about keys changed by the client itself. */ + if (target->flags & CLIENT_TRACKING_NOLOOP && + target == c) + { + continue; + } + + sendTrackingMessage(target,sdskey,sdslen(sdskey),0); } raxStop(&ri); @@ -383,6 +400,54 @@ void trackingLimitUsedSlots(void) { timeout_counter++; } +/* Generate Redis protocol for an array containing all the key names + * in the 'keys' radix tree. If the client is not NULL, the list will not + * include keys that were modified the last time by this client, in order + * to implement the NOLOOP option. + * + * If the resultin array would be empty, NULL is returned instead. */ +sds trackingBuildBroadcastReply(client *c, rax *keys) { + raxIterator ri; + uint64_t count; + + if (c == NULL) { + count = raxSize(keys); + } else { + count = 0; + raxStart(&ri,keys); + raxSeek(&ri,"^",NULL,0); + while(raxNext(&ri)) { + if (ri.data != c) count++; + } + raxStop(&ri); + + if (count == 0) return NULL; + } + + /* Create the array reply with the list of keys once, then send + * it to all the clients subscribed to this prefix. */ + char buf[32]; + size_t len = ll2string(buf,sizeof(buf),count); + sds proto = sdsempty(); + proto = sdsMakeRoomFor(proto,count*15); + proto = sdscatlen(proto,"*",1); + proto = sdscatlen(proto,buf,len); + proto = sdscatlen(proto,"\r\n",2); + raxStart(&ri,keys); + raxSeek(&ri,"^",NULL,0); + while(raxNext(&ri)) { + if (c && ri.data == c) continue; + len = ll2string(buf,sizeof(buf),ri.key_len); + proto = sdscatlen(proto,"$",1); + proto = sdscatlen(proto,buf,len); + proto = sdscatlen(proto,"\r\n",2); + proto = sdscatlen(proto,ri.key,ri.key_len); + proto = sdscatlen(proto,"\r\n",2); + } + raxStop(&ri); + return proto; +} + /* This function will run the prefixes of clients in BCAST mode and * keys that were modified about each prefix, and will send the * notifications to each client in each prefix. */ @@ -397,26 +462,10 @@ void trackingBroadcastInvalidationMessages(void) { while(raxNext(&ri)) { bcastState *bs = ri.data; if (raxSize(bs->keys)) { - /* Create the array reply with the list of keys once, then send - * it to all the clients subscribed to this prefix. */ - char buf[32]; - size_t len = ll2string(buf,sizeof(buf),raxSize(bs->keys)); - sds proto = sdsempty(); - proto = sdsMakeRoomFor(proto,raxSize(bs->keys)*15); - proto = sdscatlen(proto,"*",1); - proto = sdscatlen(proto,buf,len); - proto = sdscatlen(proto,"\r\n",2); - raxStart(&ri2,bs->keys); - raxSeek(&ri2,"^",NULL,0); - while(raxNext(&ri2)) { - len = ll2string(buf,sizeof(buf),ri2.key_len); - proto = sdscatlen(proto,"$",1); - proto = sdscatlen(proto,buf,len); - proto = sdscatlen(proto,"\r\n",2); - proto = sdscatlen(proto,ri2.key,ri2.key_len); - proto = sdscatlen(proto,"\r\n",2); - } - raxStop(&ri2); + + /* Generate the common protocol for all the clients that are + * not using the NOLOOP option. */ + sds proto = trackingBuildBroadcastReply(NULL,bs->keys); /* Send this array of keys to every client in the list. */ raxStart(&ri2,bs->clients); @@ -424,7 +473,14 @@ void trackingBroadcastInvalidationMessages(void) { while(raxNext(&ri2)) { client *c; memcpy(&c,ri2.key,sizeof(c)); - sendTrackingMessage(c,proto,sdslen(proto),1); + if (c->flags & CLIENT_TRACKING_NOLOOP) { + /* This client may have certain keys excluded. */ + sds adhoc = trackingBuildBroadcastReply(c,bs->keys); + sendTrackingMessage(c,adhoc,sdslen(adhoc),1); + sdsfree(adhoc); + } else { + sendTrackingMessage(c,proto,sdslen(proto),1); + } } raxStop(&ri2); From 9029a7ffe9a63566d4c4dbde8642a2d6f617cb66 Mon Sep 17 00:00:00 2001 From: yanhui13 Date: Tue, 21 Apr 2020 16:55:05 +0800 Subject: [PATCH 0292/1280] optimize the output of cluster slots --- src/cluster.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/cluster.c b/src/cluster.c index 2377b386b..4edc068ba 100644 --- a/src/cluster.c +++ b/src/cluster.c @@ -4191,11 +4191,17 @@ void clusterReplyMultiBulkSlots(client *c) { while((de = dictNext(di)) != NULL) { clusterNode *node = dictGetVal(de); int j = 0, start = -1; + int i, nested_elements = 0; /* Skip slaves (that are iterated when producing the output of their * master) and masters not serving any slot. */ if (!nodeIsMaster(node) || node->numslots == 0) continue; + for(i = 0; i < node->numslaves; i++) { + if (nodeFailed(node->slaves[i])) continue; + nested_elements++; + } + for (j = 0; j < CLUSTER_SLOTS; j++) { int bit, i; @@ -4203,8 +4209,7 @@ void clusterReplyMultiBulkSlots(client *c) { if (start == -1) start = j; } if (start != -1 && (!bit || j == CLUSTER_SLOTS-1)) { - int nested_elements = 3; /* slots (2) + master addr (1). */ - void *nested_replylen = addReplyDeferredLen(c); + addReplyArrayLen(c, nested_elements + 3); /* slots (2) + master addr (1). */ if (bit && j == CLUSTER_SLOTS-1) j++; @@ -4234,9 +4239,7 @@ void clusterReplyMultiBulkSlots(client *c) { addReplyBulkCString(c, node->slaves[i]->ip); addReplyLongLong(c, node->slaves[i]->port); addReplyBulkCBuffer(c, node->slaves[i]->name, CLUSTER_NAMELEN); - nested_elements++; } - setDeferredArrayLen(c, nested_replylen, nested_elements); num_masters++; } } From fc3d393607a69e2bebeccae8364feeceaf93fbd7 Mon Sep 17 00:00:00 2001 From: yanhui13 Date: Tue, 21 Apr 2020 16:56:10 +0800 Subject: [PATCH 0293/1280] add tcl test for cluster slots --- tests/cluster/tests/15-cluster-slots.tcl | 44 ++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 tests/cluster/tests/15-cluster-slots.tcl diff --git a/tests/cluster/tests/15-cluster-slots.tcl b/tests/cluster/tests/15-cluster-slots.tcl new file mode 100644 index 000000000..dc9938ef6 --- /dev/null +++ b/tests/cluster/tests/15-cluster-slots.tcl @@ -0,0 +1,44 @@ +source "../tests/includes/init-tests.tcl" + +proc cluster_allocate_mixedSlots {n} { + set slot 16383 + while {$slot >= 0} { + set node [expr {$slot % $n}] + lappend slots_$node $slot + incr slot -1 + } + for {set j 0} {$j < $n} {incr j} { + R $j cluster addslots {*}[set slots_${j}] + } +} + +proc create_cluster_with_mixedSlot {masters slaves} { + cluster_allocate_mixedSlots $masters + if {$slaves} { + cluster_allocate_slaves $masters $slaves + } + assert_cluster_state ok +} + +test "Create a 5 nodes cluster" { + create_cluster_with_mixedSlot 5 15 +} + +test "Cluster is up" { + assert_cluster_state ok +} + +test "Cluster is writable" { + cluster_write_test 0 +} + +test "Instance #5 is a slave" { + assert {[RI 5 role] eq {slave}} +} + +test "client do not break when cluster slot" { + R 0 config set client-output-buffer-limit "normal 33554432 16777216 60" + if { [catch {R 0 cluster slots}] } { + fail "output overflow when cluster slots" + } +} \ No newline at end of file From 941eb7c73e9dd437df08703b3294fd4d18d707e7 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 21 Apr 2020 17:29:18 +0200 Subject: [PATCH 0294/1280] Tracking: NOLOOP further implementation and fixes. --- src/networking.c | 2 ++ src/tracking.c | 25 +++++++++++++++++++------ 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/src/networking.c b/src/networking.c index 1f5d0bd5d..744979d16 100644 --- a/src/networking.c +++ b/src/networking.c @@ -2289,6 +2289,8 @@ NULL options |= CLIENT_TRACKING_OPTIN; } else if (!strcasecmp(c->argv[j]->ptr,"optout")) { options |= CLIENT_TRACKING_OPTOUT; + } else if (!strcasecmp(c->argv[j]->ptr,"noloop")) { + options |= CLIENT_TRACKING_NOLOOP; } else if (!strcasecmp(c->argv[j]->ptr,"prefix") && moreargs) { j++; prefix = zrealloc(prefix,sizeof(robj*)*(numprefix+1)); diff --git a/src/tracking.c b/src/tracking.c index 434e086b5..48d231627 100644 --- a/src/tracking.c +++ b/src/tracking.c @@ -94,7 +94,8 @@ void disableTracking(client *c) { server.tracking_clients--; c->flags &= ~(CLIENT_TRACKING|CLIENT_TRACKING_BROKEN_REDIR| CLIENT_TRACKING_BCAST|CLIENT_TRACKING_OPTIN| - CLIENT_TRACKING_OPTOUT|CLIENT_TRACKING_CACHING); + CLIENT_TRACKING_OPTOUT|CLIENT_TRACKING_CACHING| + CLIENT_TRACKING_NOLOOP); } } @@ -129,14 +130,19 @@ void enableTracking(client *c, uint64_t redirect_to, uint64_t options, robj **pr if (!(c->flags & CLIENT_TRACKING)) server.tracking_clients++; c->flags |= CLIENT_TRACKING; c->flags &= ~(CLIENT_TRACKING_BROKEN_REDIR|CLIENT_TRACKING_BCAST| - CLIENT_TRACKING_OPTIN|CLIENT_TRACKING_OPTOUT); + CLIENT_TRACKING_OPTIN|CLIENT_TRACKING_OPTOUT| + CLIENT_TRACKING_NOLOOP); c->client_tracking_redirection = redirect_to; + + /* This may be the first client we ever enable. Crete the tracking + * table if it does not exist. */ if (TrackingTable == NULL) { TrackingTable = raxNew(); PrefixTable = raxNew(); TrackingChannelName = createStringObject("__redis__:invalidate",20); } + /* For broadcasting, set the list of prefixes in the client. */ if (options & CLIENT_TRACKING_BCAST) { c->flags |= CLIENT_TRACKING_BCAST; if (numprefix == 0) enableBcastTrackingForPrefix(c,"",0); @@ -145,7 +151,10 @@ void enableTracking(client *c, uint64_t redirect_to, uint64_t options, robj **pr enableBcastTrackingForPrefix(c,sdsprefix,sdslen(sdsprefix)); } } - c->flags |= options & (CLIENT_TRACKING_OPTIN|CLIENT_TRACKING_OPTOUT); + + /* Set the remaining flags that don't need any special handling. */ + c->flags |= options & (CLIENT_TRACKING_OPTIN|CLIENT_TRACKING_OPTOUT| + CLIENT_TRACKING_NOLOOP); } /* This function is called after the execution of a readonly command in the @@ -459,10 +468,12 @@ void trackingBroadcastInvalidationMessages(void) { raxStart(&ri,PrefixTable); raxSeek(&ri,"^",NULL,0); + + /* For each prefix... */ while(raxNext(&ri)) { bcastState *bs = ri.data; - if (raxSize(bs->keys)) { + if (raxSize(bs->keys)) { /* Generate the common protocol for all the clients that are * not using the NOLOOP option. */ sds proto = trackingBuildBroadcastReply(NULL,bs->keys); @@ -476,8 +487,10 @@ void trackingBroadcastInvalidationMessages(void) { if (c->flags & CLIENT_TRACKING_NOLOOP) { /* This client may have certain keys excluded. */ sds adhoc = trackingBuildBroadcastReply(c,bs->keys); - sendTrackingMessage(c,adhoc,sdslen(adhoc),1); - sdsfree(adhoc); + if (adhoc) { + sendTrackingMessage(c,adhoc,sdslen(adhoc),1); + sdsfree(adhoc); + } } else { sendTrackingMessage(c,proto,sdslen(proto),1); } From 9bb3429f1a84b29d245b0b348d4ef50ef5526c46 Mon Sep 17 00:00:00 2001 From: Theo Buehler Date: Wed, 22 Apr 2020 09:43:01 +0200 Subject: [PATCH 0295/1280] TLS: Fix build with SSL_OP_NO_CLIENT_RENEGOTIATION There is no ssl in this scope, so the build breaks. All the other options are set directly on the ctx. --- src/tls.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tls.c b/src/tls.c index ea0c34469..c18aafebe 100644 --- a/src/tls.c +++ b/src/tls.c @@ -160,7 +160,7 @@ int tlsConfigure(redisTLSContextConfig *ctx_config) { #endif #ifdef SSL_OP_NO_CLIENT_RENEGOTIATION - SSL_CTX_set_options(ssl->ctx, SSL_OP_NO_CLIENT_RENEGOTIATION); + SSL_CTX_set_options(ctx, SSL_OP_NO_CLIENT_RENEGOTIATION); #endif if (ctx_config->prefer_server_ciphers) From 62a3ec88486147993c7a281a17d7408dbdfd3fd2 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 22 Apr 2020 10:49:17 +0200 Subject: [PATCH 0296/1280] Tracking: signal key as modified when evicting. --- src/evict.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/evict.c b/src/evict.c index 71260c040..a829f1f99 100644 --- a/src/evict.c +++ b/src/evict.c @@ -569,6 +569,7 @@ int freeMemoryIfNeeded(void) { dbAsyncDelete(db,keyobj); else dbSyncDelete(db,keyobj); + signalModifiedKey(NULL,db,keyobj); latencyEndMonitor(eviction_latency); latencyAddSampleIfNeeded("eviction-del",eviction_latency); latencyRemoveNestedEvent(latency,eviction_latency); From 3b98f3f2ce62d778e99820f106f484129e64468c Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 22 Apr 2020 11:24:19 +0200 Subject: [PATCH 0297/1280] Tracking: NOLOOP tests. --- tests/unit/tracking.tcl | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/tests/unit/tracking.tcl b/tests/unit/tracking.tcl index 2058319f7..dc6cd671a 100644 --- a/tests/unit/tracking.tcl +++ b/tests/unit/tracking.tcl @@ -7,6 +7,9 @@ start_server {tags {"tracking"}} { $rd1 subscribe __redis__:invalidate $rd1 read ; # Consume the SUBSCRIBE reply. + # Create another client as well in order to test NOLOOP + set rd2 [redis_deferring_client] + test {Clients are able to enable tracking and redirect it} { r CLIENT TRACKING on REDIRECT $redir } {*OK} @@ -62,5 +65,34 @@ start_server {tags {"tracking"}} { assert {$keys eq {c:1234}} } + test {Tracking NOLOOP mode in standard mode works} { + r CLIENT TRACKING off + r CLIENT TRACKING on REDIRECT $redir NOLOOP + r MGET otherkey1 loopkey otherkey2 + $rd2 SET otherkey1 1; # We should get this + r SET loopkey 1 ; # We should not get this + $rd2 SET otherkey2 1; # We should get this + # Because of the internals, we know we are going to receive + # two separated notifications for the two different prefixes. + set keys1 [lsort [lindex [$rd1 read] 2]] + set keys2 [lsort [lindex [$rd1 read] 2]] + set keys [lsort [list {*}$keys1 {*}$keys2]] + assert {$keys eq {otherkey1 otherkey2}} + } + + test {Tracking NOLOOP mode in BCAST mode works} { + r CLIENT TRACKING off + r CLIENT TRACKING on BCAST REDIRECT $redir NOLOOP + $rd2 SET otherkey1 1; # We should get this + r SET loopkey 1 ; # We should not get this + $rd2 SET otherkey2 1; # We should get this + # Because of the internals, we know we are going to receive + # two separated notifications for the two different prefixes. + set keys1 [lsort [lindex [$rd1 read] 2]] + set keys2 [lsort [lindex [$rd1 read] 2]] + set keys [lsort [list {*}$keys1 {*}$keys2]] + assert {$keys eq {otherkey1 otherkey2}} + } + $rd1 close } From 4222dfae6efd5108b9d0ed6e61a16d89a425883c Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 22 Apr 2020 11:45:34 +0200 Subject: [PATCH 0298/1280] Tracking: test expired keys notifications. --- tests/unit/tracking.tcl | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/unit/tracking.tcl b/tests/unit/tracking.tcl index dc6cd671a..43bb5f864 100644 --- a/tests/unit/tracking.tcl +++ b/tests/unit/tracking.tcl @@ -94,5 +94,18 @@ start_server {tags {"tracking"}} { assert {$keys eq {otherkey1 otherkey2}} } + test {Tracking gets notification of expired keys} { + r CLIENT TRACKING off + r CLIENT TRACKING on BCAST REDIRECT $redir NOLOOP + r SET mykey myval px 1 + r SET mykeyotherkey myval ; # We should not get it + after 1000 + # Because of the internals, we know we are going to receive + # two separated notifications for the two different prefixes. + set keys1 [lsort [lindex [$rd1 read] 2]] + set keys [lsort [list {*}$keys1]] + assert {$keys eq {mykey}} + } + $rd1 close } From 3df7a7dcae0cb363b06cec11d596daa707fa5921 Mon Sep 17 00:00:00 2001 From: Valentino Geron Date: Tue, 21 Apr 2020 20:55:43 +0300 Subject: [PATCH 0299/1280] XREADGROUP with NOACK should propagate only one XGROUP SETID command --- src/t_stream.c | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/src/t_stream.c b/src/t_stream.c index 155167af9..2dffdbcdf 100644 --- a/src/t_stream.c +++ b/src/t_stream.c @@ -935,6 +935,8 @@ size_t streamReplyWithRange(client *c, stream *s, streamID *start, streamID *end streamIterator si; int64_t numfields; streamID id; + int propagate_last_id = 0; + int noack = flags & STREAM_RWR_NOACK; /* If the client is asking for some history, we serve it using a * different function, so that we return entries *solely* from its @@ -950,12 +952,14 @@ size_t streamReplyWithRange(client *c, stream *s, streamID *start, streamID *end arraylen_ptr = addReplyDeferredLen(c); streamIteratorStart(&si,s,start,end,rev); while(streamIteratorGetID(&si,&id,&numfields)) { - int propagate_last_id = 0; - /* Update the group last_id if needed. */ if (group && streamCompareID(&id,&group->last_id) > 0) { group->last_id = id; - propagate_last_id = 1; + /* Group last id should be propagated only if NOACK was + * specified, otherwise the last id would be included + * in XCLAIM. */ + if (noack) + propagate_last_id = 1; } /* Emit a two elements array for each item. The first is @@ -983,7 +987,7 @@ size_t streamReplyWithRange(client *c, stream *s, streamID *start, streamID *end * XGROUP SETID command. So if we find that there is already * a NACK for the entry, we need to associate it to the new * consumer. */ - if (group && !(flags & STREAM_RWR_NOACK)) { + if (group && !noack) { unsigned char buf[sizeof(streamID)]; streamEncodeID(buf,&id); @@ -1020,14 +1024,16 @@ size_t streamReplyWithRange(client *c, stream *s, streamID *start, streamID *end streamPropagateXCLAIM(c,spi->keyname,group,spi->groupname,idarg,nack); decrRefCount(idarg); } - } else { - if (propagate_last_id) - streamPropagateGroupID(c,spi->keyname,group,spi->groupname); } arraylen++; if (count && count == arraylen) break; } + + if (spi && propagate_last_id) { + streamPropagateGroupID(c,spi->keyname,group,spi->groupname); + } + streamIteratorStop(&si); if (arraylen_ptr) setDeferredArrayLen(c,arraylen_ptr,arraylen); return arraylen; From febfd6a3040ed4ba0736a28541a85805c100fee4 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 22 Apr 2020 17:14:15 +0200 Subject: [PATCH 0300/1280] ACL: deny commands execution of disabled users. --- src/acl.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/acl.c b/src/acl.c index 75b954c5e..9e2ed6af7 100644 --- a/src/acl.c +++ b/src/acl.c @@ -1055,6 +1055,10 @@ int ACLCheckCommandPerm(client *c, int *keyidxptr) { /* If there is no associated user, the connection can run anything. */ if (u == NULL) return ACL_OK; + /* If the user is disabled we don't allow the execution of any + * command. */ + if (!(u->flags & USER_FLAG_ENABLED)) return ACL_DENIED_CMD; + /* Check if the user can execute this command. */ if (!(u->flags & USER_FLAG_ALLCOMMANDS) && c->cmd->proc != authCommand) From 530a73e8ccf2a7259bbfda9c6aa157c3b7054512 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 23 Apr 2020 10:39:53 +0200 Subject: [PATCH 0301/1280] ACL GENPASS: emit 256 bits instead of 128. --- src/acl.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/acl.c b/src/acl.c index 9e2ed6af7..228811cba 100644 --- a/src/acl.c +++ b/src/acl.c @@ -1819,7 +1819,7 @@ void aclCommand(client *c) { dictReleaseIterator(di); setDeferredArrayLen(c,dl,arraylen); } else if (!strcasecmp(sub,"genpass") && c->argc == 2) { - char pass[32]; /* 128 bits of actual pseudo random data. */ + char pass[64]; /* 256 bits of actual pseudo random data. */ getRandomHexChars(pass,sizeof(pass)); addReplyBulkCBuffer(c,pass,sizeof(pass)); } else if (!strcasecmp(sub,"log") && (c->argc == 2 || c->argc ==3)) { From 7cedce1171833465ba8af584c2e4656b1528fdb1 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 23 Apr 2020 10:53:21 +0200 Subject: [PATCH 0302/1280] ACL GENPASS: take number of bits as argument. --- src/acl.c | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/src/acl.c b/src/acl.c index 228811cba..bf5dd18f1 100644 --- a/src/acl.c +++ b/src/acl.c @@ -1626,7 +1626,7 @@ void addACLLogEntry(client *c, int reason, int keypos, sds username) { * ACL SETUSER ... acl rules ... * ACL DELUSER [...] * ACL GETUSER - * ACL GENPASS + * ACL GENPASS [] * ACL WHOAMI * ACL LOG [ | RESET] */ @@ -1818,10 +1818,25 @@ void aclCommand(client *c) { } dictReleaseIterator(di); setDeferredArrayLen(c,dl,arraylen); - } else if (!strcasecmp(sub,"genpass") && c->argc == 2) { - char pass[64]; /* 256 bits of actual pseudo random data. */ - getRandomHexChars(pass,sizeof(pass)); - addReplyBulkCBuffer(c,pass,sizeof(pass)); + } else if (!strcasecmp(sub,"genpass") && (c->argc == 2 || c->argc == 3)) { + #define GENPASS_MAX_BITS 4096 + char pass[GENPASS_MAX_BITS/8*2]; /* Hex representation. */ + long bits = 256; /* By default generate 256 bits passwords. */ + + if (c->argc == 3 && getLongFromObjectOrReply(c,c->argv[2],&bits,NULL) + != C_OK) return; + + if (bits <= 0 || bits > GENPASS_MAX_BITS) { + addReplyErrorFormat(c, + "ACL GENPASS argument must be the number of " + "bits for the output password, a positive number " + "up to %d",GENPASS_MAX_BITS); + return; + } + + long chars = (bits+3)/4; /* Round to number of characters to emit. */ + getRandomHexChars(pass,chars); + addReplyBulkCBuffer(c,pass,chars); } else if (!strcasecmp(sub,"log") && (c->argc == 2 || c->argc ==3)) { long count = 10; /* Number of entries to emit by default. */ @@ -1899,7 +1914,7 @@ void aclCommand(client *c) { "DELUSER [...] -- Delete a list of users.", "CAT -- List available categories.", "CAT -- List commands inside category.", -"GENPASS -- Generate a secure user password.", +"GENPASS [] -- Generate a secure user password.", "WHOAMI -- Return the current connection username.", "LOG [ | RESET] -- Show the ACL log entries.", NULL From aede1b04c9e143146e25d3afb7a2d3bd3a84b668 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 23 Apr 2020 11:17:42 +0200 Subject: [PATCH 0303/1280] getRandomBytes(): use HMAC-SHA256. Now that we have an interface to use this API directly, via ACL GENPASS, we are no longer sure what people could do with it. So why don't make it a strong primitive exported by Redis in order to create unique IDs and so forth? The implementation was tested against the test vectors that can be found in RFC4231. --- src/util.c | 38 +++++++++++++++++++++++++++++--------- 1 file changed, 29 insertions(+), 9 deletions(-) diff --git a/src/util.c b/src/util.c index bd8f0fb98..d173f776f 100644 --- a/src/util.c +++ b/src/util.c @@ -42,7 +42,7 @@ #include #include "util.h" -#include "sha1.h" +#include "sha256.h" /* Glob-style pattern matching. */ int stringmatchlen(const char *pattern, int patternLen, @@ -622,7 +622,7 @@ int ld2string(char *buf, size_t len, long double value, ld2string_mode mode) { void getRandomBytes(unsigned char *p, size_t len) { /* Global state. */ static int seed_initialized = 0; - static unsigned char seed[20]; /* The SHA1 seed, from /dev/urandom. */ + static unsigned char seed[64]; /* 512 bit internal block size. */ static uint64_t counter = 0; /* The counter we hash with the seed. */ if (!seed_initialized) { @@ -647,14 +647,34 @@ void getRandomBytes(unsigned char *p, size_t len) { } while(len) { - unsigned char digest[20]; - SHA1_CTX ctx; - unsigned int copylen = len > 20 ? 20 : len; + /* This implements SHA256-HMAC. */ + unsigned char digest[SHA256_BLOCK_SIZE]; + unsigned char kxor[64]; + unsigned int copylen = + len > SHA256_BLOCK_SIZE ? SHA256_BLOCK_SIZE : len; - SHA1Init(&ctx); - SHA1Update(&ctx, seed, sizeof(seed)); - SHA1Update(&ctx, (unsigned char*)&counter,sizeof(counter)); - SHA1Final(digest, &ctx); + /* IKEY: key xored with 0x36. */ + memcpy(kxor,seed,sizeof(kxor)); + for (unsigned int i = 0; i < sizeof(kxor); i++) kxor[i] ^= 0x36; + + /* Obtain HASH(IKEY||MESSAGE). */ + SHA256_CTX ctx; + sha256_init(&ctx); + sha256_update(&ctx,kxor,sizeof(kxor)); + sha256_update(&ctx,(unsigned char*)&counter,sizeof(counter)); + sha256_final(&ctx,digest); + + /* OKEY: key xored with 0x5c. */ + memcpy(kxor,seed,sizeof(kxor)); + for (unsigned int i = 0; i < sizeof(kxor); i++) kxor[i] ^= 0x5C; + + /* Obtain HASH(OKEY || HASH(IKEY||MESSAGE)). */ + sha256_init(&ctx); + sha256_update(&ctx,kxor,sizeof(kxor)); + sha256_update(&ctx,digest,SHA256_BLOCK_SIZE); + sha256_final(&ctx,digest); + + /* Increment the counter for the next iteration. */ counter++; memcpy(p,digest,copylen); From 8f0c7139b81f5ea4b55a66e22f41a52ebd0ed7cf Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 23 Apr 2020 11:56:36 +0200 Subject: [PATCH 0304/1280] ACL: re-enable command execution of disabled users. After all I changed idea again: enabled/disabled should have a more clear meaning, and it only means: you can't authenticate with such user with new connections, however old connections continue to work as expected. --- src/acl.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/acl.c b/src/acl.c index bf5dd18f1..3194feb5b 100644 --- a/src/acl.c +++ b/src/acl.c @@ -1055,10 +1055,6 @@ int ACLCheckCommandPerm(client *c, int *keyidxptr) { /* If there is no associated user, the connection can run anything. */ if (u == NULL) return ACL_OK; - /* If the user is disabled we don't allow the execution of any - * command. */ - if (!(u->flags & USER_FLAG_ENABLED)) return ACL_DENIED_CMD; - /* Check if the user can execute this command. */ if (!(u->flags & USER_FLAG_ALLCOMMANDS) && c->cmd->proc != authCommand) From 15e36d86745028820fcad6733d1bb46959b07730 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 23 Apr 2020 16:13:45 +0200 Subject: [PATCH 0305/1280] Minor aesthetic changes to #7135. --- src/t_stream.c | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/src/t_stream.c b/src/t_stream.c index 8abbee02f..596d8ad0d 100644 --- a/src/t_stream.c +++ b/src/t_stream.c @@ -955,11 +955,10 @@ size_t streamReplyWithRange(client *c, stream *s, streamID *start, streamID *end /* Update the group last_id if needed. */ if (group && streamCompareID(&id,&group->last_id) > 0) { group->last_id = id; - /* Group last id should be propagated only if NOACK was - * specified, otherwise the last id would be included - * in XCLAIM. */ - if (noack) - propagate_last_id = 1; + /* Group last ID should be propagated only if NOACK was + * specified, otherwise the last id will be included + * in the propagation of XCLAIM itself. */ + if (noack) propagate_last_id = 1; } /* Emit a two elements array for each item. The first is @@ -1030,9 +1029,8 @@ size_t streamReplyWithRange(client *c, stream *s, streamID *start, streamID *end if (count && count == arraylen) break; } - if (spi && propagate_last_id) { + if (spi && propagate_last_id) streamPropagateGroupID(c,spi->keyname,group,spi->groupname); - } streamIteratorStop(&si); if (arraylen_ptr) setDeferredArrayLen(c,arraylen_ptr,arraylen); From e8852b5cc53e4ca61461ed38f2c5e66c4659b977 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 24 Apr 2020 10:13:20 +0200 Subject: [PATCH 0306/1280] Also use propagate() in streamPropagateGroupID(). --- src/t_stream.c | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/t_stream.c b/src/t_stream.c index 596d8ad0d..758db637e 100644 --- a/src/t_stream.c +++ b/src/t_stream.c @@ -848,6 +848,11 @@ void streamPropagateXCLAIM(client *c, robj *key, streamCG *group, robj *groupnam argv[11] = createStringObject("JUSTID",6); argv[12] = createStringObject("LASTID",6); argv[13] = createObjectFromStreamID(&group->last_id); + + /* We use progagate() because this code path is not always called from + * the command execution context. Moreover this will just alter the + * consumer group state, and we don't need MULTI/EXEC wrapping because + * there is no message state cross-message atomicity required. */ propagate(server.xclaimCommand,c->db->id,argv,14,PROPAGATE_AOF|PROPAGATE_REPL); decrRefCount(argv[0]); decrRefCount(argv[3]); @@ -875,7 +880,12 @@ void streamPropagateGroupID(client *c, robj *key, streamCG *group, robj *groupna argv[2] = key; argv[3] = groupname; argv[4] = createObjectFromStreamID(&group->last_id); - alsoPropagate(server.xgroupCommand,c->db->id,argv,5,PROPAGATE_AOF|PROPAGATE_REPL); + + /* We use progagate() because this code path is not always called from + * the command execution context. Moreover this will just alter the + * consumer group state, and we don't need MULTI/EXEC wrapping because + * there is no message state cross-message atomicity required. */ + propagate(server.xgroupCommand,c->db->id,argv,5,PROPAGATE_AOF|PROPAGATE_REPL); decrRefCount(argv[0]); decrRefCount(argv[1]); decrRefCount(argv[4]); From 4ed5b7cb74caf5bef6606909603e371af0da4f9b Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Fri, 24 Apr 2020 17:20:28 +0300 Subject: [PATCH 0307/1280] optimize memory usage of deferred replies When deffered reply is added the previous reply node cannot be used so all the extra space we allocated in it is wasted. in case someone uses deffered replies in a loop, each time adding a small reply, each of these reply nodes (the small string reply) would have consumed a 16k block. now when we add anther diferred reply node, we trim the unused portion of the previous reply block. see #7123 --- src/networking.c | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/src/networking.c b/src/networking.c index 744979d16..a4247a8a6 100644 --- a/src/networking.c +++ b/src/networking.c @@ -436,6 +436,36 @@ void addReplyStatusFormat(client *c, const char *fmt, ...) { sdsfree(s); } +/* Sometimes we are forced to create a new reply node, and we can't append to + * the previous one, when that happens, we wanna try to trim the unused space + * at the end of the last reply node which we won't use anymore. */ +void trimReplyUnusedTailSpace(client *c) { + listNode *ln = listLast(c->reply); + clientReplyBlock *tail = ln? listNodeValue(ln): NULL; + + /* Note that 'tail' may be NULL even if we have a tail node, becuase when + * addDeferredMultiBulkLength() is used */ + if (!tail) return; + + /* We only try to trim the space is relatively high (more than a 1/4 of the + * allocation), otherwise there's a high chance realloc will NOP. + * Also, to avoid large memmove which happens as part of realloc, we only do + * that if the used part is small. */ + if (tail->size - tail->used > tail->size / 4 && + tail->used < PROTO_REPLY_CHUNK_BYTES) + { + size_t old_size = tail->size; + tail = zrealloc(tail, tail->used + sizeof(clientReplyBlock)); + /* If realloc was a NOP, we got the same value which has internal frag */ + if (tail == listNodeValue(ln)) return; + /* take over the allocation's internal fragmentation (at least for + * memory usage tracking) */ + tail->size = zmalloc_usable(tail) - sizeof(clientReplyBlock); + c->reply_bytes += tail->size - old_size; + listNodeValue(ln) = tail; + } +} + /* Adds an empty object to the reply list that will contain the multi bulk * length, which is not known when this function is called. */ void *addReplyDeferredLen(client *c) { @@ -443,6 +473,7 @@ void *addReplyDeferredLen(client *c) { * ready to be sent, since we are sure that before returning to the * event loop setDeferredAggregateLen() will be called. */ if (prepareClientToWrite(c) != C_OK) return NULL; + trimReplyUnusedTailSpace(c); listAddNodeTail(c->reply,NULL); /* NULL is our placeholder. */ return listLast(c->reply); } From ae3cf7911a85ce0faa5ebf84d5d3b1ea92141537 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 24 Apr 2020 16:49:27 +0200 Subject: [PATCH 0308/1280] LCS -> STRALGO LCS. STRALGO should be a container for mostly read-only string algorithms in Redis. The algorithms should have two main characteristics: 1. They should be non trivial to compute, and often not part of programming language standard libraries. 2. They should be fast enough that it is a good idea to have optimized C implementations. Next thing I would love to see? A small strings compression algorithm. --- src/server.c | 2 +- src/server.h | 2 +- src/t_string.c | 23 ++++++++++++++++++----- tests/unit/type/string.tcl | 16 ++++++++-------- 4 files changed, 28 insertions(+), 15 deletions(-) diff --git a/src/server.c b/src/server.c index fc9b87aae..f7af79c3f 100644 --- a/src/server.c +++ b/src/server.c @@ -1006,7 +1006,7 @@ struct redisCommand redisCommandTable[] = { "admin no-script no-slowlog ok-loading ok-stale", 0,NULL,0,0,0,0,0,0}, - {"lcs",lcsCommand,-4, + {"stralgo",stralgoCommand,-2, "write use-memory @string", 0,lcsGetKeys,0,0,0,0,0,0} }; diff --git a/src/server.h b/src/server.h index d39359dce..9e1e506af 100644 --- a/src/server.h +++ b/src/server.h @@ -2389,7 +2389,7 @@ void xdelCommand(client *c); void xtrimCommand(client *c); void lolwutCommand(client *c); void aclCommand(client *c); -void lcsCommand(client *c); +void stralgoCommand(client *c); #if defined(__GNUC__) void *calloc(size_t count, size_t size) __attribute__ ((deprecated)); diff --git a/src/t_string.c b/src/t_string.c index ef382bb0c..d4eb04769 100644 --- a/src/t_string.c +++ b/src/t_string.c @@ -480,18 +480,31 @@ void strlenCommand(client *c) { addReplyLongLong(c,stringObjectLen(o)); } -/* LCS -- Longest common subsequence. + +/* STRALGO -- Implement complex algorithms on strings. * - * LCS [IDX] [MINMATCHLEN ] - * STRINGS | KEYS */ -void lcsCommand(client *c) { + * STRALGO ... arguments ... */ +void stralgoLCS(client *c); /* This implements the LCS algorithm. */ +void stralgoCommand(client *c) { + /* Select the algorithm. */ + if (!strcasecmp(c->argv[1]->ptr,"lcs")) { + stralgoLCS(c); + } else { + addReply(c,shared.syntaxerr); + } +} + +/* STRALGO [IDX] [MINMATCHLEN ] [WITHMATCHLEN] + * STRINGS | KEYS + */ +void stralgoLCS(client *c) { uint32_t i, j; long long minmatchlen = 0; sds a = NULL, b = NULL; int getlen = 0, getidx = 0, withmatchlen = 0; robj *obja = NULL, *objb = NULL; - for (j = 1; j < (uint32_t)c->argc; j++) { + for (j = 2; j < (uint32_t)c->argc; j++) { char *opt = c->argv[j]->ptr; int moreargs = (c->argc-1) - j; diff --git a/tests/unit/type/string.tcl b/tests/unit/type/string.tcl index b9ef9de7a..8126cdee8 100644 --- a/tests/unit/type/string.tcl +++ b/tests/unit/type/string.tcl @@ -424,29 +424,29 @@ start_server {tags {"string"}} { set rna2 {ATTAAAGGTTTATACCTTCCCAGGTAACAAACCAACCAACTTTCGATCTCTTGTAGATCTGTTCTCTAAACGAACTTTAAAATCTGTGTGGCTGTCACTCGGCTGCATGCTTAGTGCACTCACGCAGTATAATTAATAACTAATTACTGTCGTTGACAGGACACGAGTAACTCGTCTATCTTCTGCAGGCTGCTTACGGTTTCGTCCGTGTTGCAGCCGATCATCAGCACATCTAGGTTT} set rnalcs {ACCTTCCCAGGTAACAAACCAACCAACTTTCGATCTCTTGTAGATCTGTTCTCTAAACGAACTTTAAAATCTGTGTGGCTGTCACTCGGCTGCATGCTTAGTGCACTCACGCAGTATAATTAATAACTAATTACTGTCGTTGACAGGACACGAGTAACTCGTCTATCTTCTGCAGGCTGCTTACGGTTTCGTCCGTGTTGCAGCCGATCATCAGCACATCTAGGTTT} - test {LCS string output with STRINGS option} { - r LCS STRINGS $rna1 $rna2 + test {STRALGO LCS string output with STRINGS option} { + r STRALGO LCS STRINGS $rna1 $rna2 } $rnalcs - test {LCS len} { - r LCS LEN STRINGS $rna1 $rna2 + test {STRALGO LCS len} { + r STRALGO LCS LEN STRINGS $rna1 $rna2 } [string length $rnalcs] test {LCS with KEYS option} { r set virus1 $rna1 r set virus2 $rna2 - r LCS KEYS virus1 virus2 + r STRALGO LCS KEYS virus1 virus2 } $rnalcs test {LCS indexes} { - dict get [r LCS IDX KEYS virus1 virus2] matches + dict get [r STRALGO LCS IDX KEYS virus1 virus2] matches } {{{238 238} {239 239}} {{236 236} {238 238}} {{229 230} {236 237}} {{224 224} {235 235}} {{1 222} {13 234}}} test {LCS indexes with match len} { - dict get [r LCS IDX KEYS virus1 virus2 WITHMATCHLEN] matches + dict get [r STRALGO LCS IDX KEYS virus1 virus2 WITHMATCHLEN] matches } {{{238 238} {239 239} 1} {{236 236} {238 238} 1} {{229 230} {236 237} 2} {{224 224} {235 235} 1} {{1 222} {13 234} 222}} test {LCS indexes with match len and minimum match len} { - dict get [r LCS IDX KEYS virus1 virus2 WITHMATCHLEN MINMATCHLEN 5] matches + dict get [r STRALGO LCS IDX KEYS virus1 virus2 WITHMATCHLEN MINMATCHLEN 5] matches } {{{1 222} {13 234} 222}} } From 0978a60b3b267f9ffdf49139630feb1c5d71b9cc Mon Sep 17 00:00:00 2001 From: Madelyn Olson Date: Fri, 24 Apr 2020 16:59:24 -0700 Subject: [PATCH 0309/1280] Implemented CRC64 based on slice by 4 --- src/Makefile | 4 +- src/crc64.c | 270 ++++++++++++++++++++++----------------------------- src/crc64.h | 1 + src/rdb.c | 3 +- src/server.c | 1 + 5 files changed, 123 insertions(+), 156 deletions(-) diff --git a/src/Makefile b/src/Makefile index 3f982cc8e..f9922afce 100644 --- a/src/Makefile +++ b/src/Makefile @@ -206,9 +206,9 @@ endif REDIS_SERVER_NAME=redis-server REDIS_SENTINEL_NAME=redis-sentinel -REDIS_SERVER_OBJ=adlist.o quicklist.o ae.o anet.o dict.o server.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o cluster.o crc16.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o memtest.o crc64.o bitops.o sentinel.o notify.o setproctitle.o blocked.o hyperloglog.o latency.o sparkline.o redis-check-rdb.o redis-check-aof.o geo.o lazyfree.o module.o evict.o expire.o geohash.o geohash_helper.o childinfo.o defrag.o siphash.o rax.o t_stream.o listpack.o localtime.o lolwut.o lolwut5.o lolwut6.o acl.o gopher.o tracking.o connection.o tls.o sha256.o timeout.o +REDIS_SERVER_OBJ=adlist.o quicklist.o ae.o anet.o dict.o server.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o cluster.o crc16.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o memtest.o crcspeed.o crc64.o bitops.o sentinel.o notify.o setproctitle.o blocked.o hyperloglog.o latency.o sparkline.o redis-check-rdb.o redis-check-aof.o geo.o lazyfree.o module.o evict.o expire.o geohash.o geohash_helper.o childinfo.o defrag.o siphash.o rax.o t_stream.o listpack.o localtime.o lolwut.o lolwut5.o lolwut6.o acl.o gopher.o tracking.o connection.o tls.o sha256.o timeout.o REDIS_CLI_NAME=redis-cli -REDIS_CLI_OBJ=anet.o adlist.o dict.o redis-cli.o zmalloc.o release.o ae.o crc64.o siphash.o crc16.o +REDIS_CLI_OBJ=anet.o adlist.o dict.o redis-cli.o zmalloc.o release.o ae.o crcspeed.o crc64.o siphash.o crc16.o REDIS_BENCHMARK_NAME=redis-benchmark REDIS_BENCHMARK_OBJ=ae.o anet.o redis-benchmark.o adlist.o dict.o zmalloc.o siphash.o REDIS_CHECK_RDB_NAME=redis-check-rdb diff --git a/src/crc64.c b/src/crc64.c index f1f764922..165941dea 100644 --- a/src/crc64.c +++ b/src/crc64.c @@ -1,16 +1,5 @@ -/* Redis uses the CRC64 variant with "Jones" coefficients and init value of 0. - * - * Specification of this CRC64 variant follows: - * Name: crc-64-jones - * Width: 64 bites - * Poly: 0xad93d23594c935a9 - * Reflected In: True - * Xor_In: 0xffffffffffffffff - * Reflected_Out: True - * Xor_Out: 0x0 - * Check("123456789"): 0xe9c6d914c4b8d9ca - * - * Copyright (c) 2012, Salvatore Sanfilippo +/* Copyright (c) 2014, Matt Stancliff + * Copyright (c) 2020, Amazon Web Services * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -37,159 +26,134 @@ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ -#include +#include "crc64.h" +#include "crcspeed.h" +static uint64_t crc64_table[8][256] = {{0}}; -static const uint64_t crc64_tab[256] = { - UINT64_C(0x0000000000000000), UINT64_C(0x7ad870c830358979), - UINT64_C(0xf5b0e190606b12f2), UINT64_C(0x8f689158505e9b8b), - UINT64_C(0xc038e5739841b68f), UINT64_C(0xbae095bba8743ff6), - UINT64_C(0x358804e3f82aa47d), UINT64_C(0x4f50742bc81f2d04), - UINT64_C(0xab28ecb46814fe75), UINT64_C(0xd1f09c7c5821770c), - UINT64_C(0x5e980d24087fec87), UINT64_C(0x24407dec384a65fe), - UINT64_C(0x6b1009c7f05548fa), UINT64_C(0x11c8790fc060c183), - UINT64_C(0x9ea0e857903e5a08), UINT64_C(0xe478989fa00bd371), - UINT64_C(0x7d08ff3b88be6f81), UINT64_C(0x07d08ff3b88be6f8), - UINT64_C(0x88b81eabe8d57d73), UINT64_C(0xf2606e63d8e0f40a), - UINT64_C(0xbd301a4810ffd90e), UINT64_C(0xc7e86a8020ca5077), - UINT64_C(0x4880fbd87094cbfc), UINT64_C(0x32588b1040a14285), - UINT64_C(0xd620138fe0aa91f4), UINT64_C(0xacf86347d09f188d), - UINT64_C(0x2390f21f80c18306), UINT64_C(0x594882d7b0f40a7f), - UINT64_C(0x1618f6fc78eb277b), UINT64_C(0x6cc0863448deae02), - UINT64_C(0xe3a8176c18803589), UINT64_C(0x997067a428b5bcf0), - UINT64_C(0xfa11fe77117cdf02), UINT64_C(0x80c98ebf2149567b), - UINT64_C(0x0fa11fe77117cdf0), UINT64_C(0x75796f2f41224489), - UINT64_C(0x3a291b04893d698d), UINT64_C(0x40f16bccb908e0f4), - UINT64_C(0xcf99fa94e9567b7f), UINT64_C(0xb5418a5cd963f206), - UINT64_C(0x513912c379682177), UINT64_C(0x2be1620b495da80e), - UINT64_C(0xa489f35319033385), UINT64_C(0xde51839b2936bafc), - UINT64_C(0x9101f7b0e12997f8), UINT64_C(0xebd98778d11c1e81), - UINT64_C(0x64b116208142850a), UINT64_C(0x1e6966e8b1770c73), - UINT64_C(0x8719014c99c2b083), UINT64_C(0xfdc17184a9f739fa), - UINT64_C(0x72a9e0dcf9a9a271), UINT64_C(0x08719014c99c2b08), - UINT64_C(0x4721e43f0183060c), UINT64_C(0x3df994f731b68f75), - UINT64_C(0xb29105af61e814fe), UINT64_C(0xc849756751dd9d87), - UINT64_C(0x2c31edf8f1d64ef6), UINT64_C(0x56e99d30c1e3c78f), - UINT64_C(0xd9810c6891bd5c04), UINT64_C(0xa3597ca0a188d57d), - UINT64_C(0xec09088b6997f879), UINT64_C(0x96d1784359a27100), - UINT64_C(0x19b9e91b09fcea8b), UINT64_C(0x636199d339c963f2), - UINT64_C(0xdf7adabd7a6e2d6f), UINT64_C(0xa5a2aa754a5ba416), - UINT64_C(0x2aca3b2d1a053f9d), UINT64_C(0x50124be52a30b6e4), - UINT64_C(0x1f423fcee22f9be0), UINT64_C(0x659a4f06d21a1299), - UINT64_C(0xeaf2de5e82448912), UINT64_C(0x902aae96b271006b), - UINT64_C(0x74523609127ad31a), UINT64_C(0x0e8a46c1224f5a63), - UINT64_C(0x81e2d7997211c1e8), UINT64_C(0xfb3aa75142244891), - UINT64_C(0xb46ad37a8a3b6595), UINT64_C(0xceb2a3b2ba0eecec), - UINT64_C(0x41da32eaea507767), UINT64_C(0x3b024222da65fe1e), - UINT64_C(0xa2722586f2d042ee), UINT64_C(0xd8aa554ec2e5cb97), - UINT64_C(0x57c2c41692bb501c), UINT64_C(0x2d1ab4dea28ed965), - UINT64_C(0x624ac0f56a91f461), UINT64_C(0x1892b03d5aa47d18), - UINT64_C(0x97fa21650afae693), UINT64_C(0xed2251ad3acf6fea), - UINT64_C(0x095ac9329ac4bc9b), UINT64_C(0x7382b9faaaf135e2), - UINT64_C(0xfcea28a2faafae69), UINT64_C(0x8632586aca9a2710), - UINT64_C(0xc9622c4102850a14), UINT64_C(0xb3ba5c8932b0836d), - UINT64_C(0x3cd2cdd162ee18e6), UINT64_C(0x460abd1952db919f), - UINT64_C(0x256b24ca6b12f26d), UINT64_C(0x5fb354025b277b14), - UINT64_C(0xd0dbc55a0b79e09f), UINT64_C(0xaa03b5923b4c69e6), - UINT64_C(0xe553c1b9f35344e2), UINT64_C(0x9f8bb171c366cd9b), - UINT64_C(0x10e3202993385610), UINT64_C(0x6a3b50e1a30ddf69), - UINT64_C(0x8e43c87e03060c18), UINT64_C(0xf49bb8b633338561), - UINT64_C(0x7bf329ee636d1eea), UINT64_C(0x012b592653589793), - UINT64_C(0x4e7b2d0d9b47ba97), UINT64_C(0x34a35dc5ab7233ee), - UINT64_C(0xbbcbcc9dfb2ca865), UINT64_C(0xc113bc55cb19211c), - UINT64_C(0x5863dbf1e3ac9dec), UINT64_C(0x22bbab39d3991495), - UINT64_C(0xadd33a6183c78f1e), UINT64_C(0xd70b4aa9b3f20667), - UINT64_C(0x985b3e827bed2b63), UINT64_C(0xe2834e4a4bd8a21a), - UINT64_C(0x6debdf121b863991), UINT64_C(0x1733afda2bb3b0e8), - UINT64_C(0xf34b37458bb86399), UINT64_C(0x8993478dbb8deae0), - UINT64_C(0x06fbd6d5ebd3716b), UINT64_C(0x7c23a61ddbe6f812), - UINT64_C(0x3373d23613f9d516), UINT64_C(0x49aba2fe23cc5c6f), - UINT64_C(0xc6c333a67392c7e4), UINT64_C(0xbc1b436e43a74e9d), - UINT64_C(0x95ac9329ac4bc9b5), UINT64_C(0xef74e3e19c7e40cc), - UINT64_C(0x601c72b9cc20db47), UINT64_C(0x1ac40271fc15523e), - UINT64_C(0x5594765a340a7f3a), UINT64_C(0x2f4c0692043ff643), - UINT64_C(0xa02497ca54616dc8), UINT64_C(0xdafce7026454e4b1), - UINT64_C(0x3e847f9dc45f37c0), UINT64_C(0x445c0f55f46abeb9), - UINT64_C(0xcb349e0da4342532), UINT64_C(0xb1eceec59401ac4b), - UINT64_C(0xfebc9aee5c1e814f), UINT64_C(0x8464ea266c2b0836), - UINT64_C(0x0b0c7b7e3c7593bd), UINT64_C(0x71d40bb60c401ac4), - UINT64_C(0xe8a46c1224f5a634), UINT64_C(0x927c1cda14c02f4d), - UINT64_C(0x1d148d82449eb4c6), UINT64_C(0x67ccfd4a74ab3dbf), - UINT64_C(0x289c8961bcb410bb), UINT64_C(0x5244f9a98c8199c2), - UINT64_C(0xdd2c68f1dcdf0249), UINT64_C(0xa7f41839ecea8b30), - UINT64_C(0x438c80a64ce15841), UINT64_C(0x3954f06e7cd4d138), - UINT64_C(0xb63c61362c8a4ab3), UINT64_C(0xcce411fe1cbfc3ca), - UINT64_C(0x83b465d5d4a0eece), UINT64_C(0xf96c151de49567b7), - UINT64_C(0x76048445b4cbfc3c), UINT64_C(0x0cdcf48d84fe7545), - UINT64_C(0x6fbd6d5ebd3716b7), UINT64_C(0x15651d968d029fce), - UINT64_C(0x9a0d8ccedd5c0445), UINT64_C(0xe0d5fc06ed698d3c), - UINT64_C(0xaf85882d2576a038), UINT64_C(0xd55df8e515432941), - UINT64_C(0x5a3569bd451db2ca), UINT64_C(0x20ed197575283bb3), - UINT64_C(0xc49581ead523e8c2), UINT64_C(0xbe4df122e51661bb), - UINT64_C(0x3125607ab548fa30), UINT64_C(0x4bfd10b2857d7349), - UINT64_C(0x04ad64994d625e4d), UINT64_C(0x7e7514517d57d734), - UINT64_C(0xf11d85092d094cbf), UINT64_C(0x8bc5f5c11d3cc5c6), - UINT64_C(0x12b5926535897936), UINT64_C(0x686de2ad05bcf04f), - UINT64_C(0xe70573f555e26bc4), UINT64_C(0x9ddd033d65d7e2bd), - UINT64_C(0xd28d7716adc8cfb9), UINT64_C(0xa85507de9dfd46c0), - UINT64_C(0x273d9686cda3dd4b), UINT64_C(0x5de5e64efd965432), - UINT64_C(0xb99d7ed15d9d8743), UINT64_C(0xc3450e196da80e3a), - UINT64_C(0x4c2d9f413df695b1), UINT64_C(0x36f5ef890dc31cc8), - UINT64_C(0x79a59ba2c5dc31cc), UINT64_C(0x037deb6af5e9b8b5), - UINT64_C(0x8c157a32a5b7233e), UINT64_C(0xf6cd0afa9582aa47), - UINT64_C(0x4ad64994d625e4da), UINT64_C(0x300e395ce6106da3), - UINT64_C(0xbf66a804b64ef628), UINT64_C(0xc5bed8cc867b7f51), - UINT64_C(0x8aeeace74e645255), UINT64_C(0xf036dc2f7e51db2c), - UINT64_C(0x7f5e4d772e0f40a7), UINT64_C(0x05863dbf1e3ac9de), - UINT64_C(0xe1fea520be311aaf), UINT64_C(0x9b26d5e88e0493d6), - UINT64_C(0x144e44b0de5a085d), UINT64_C(0x6e963478ee6f8124), - UINT64_C(0x21c640532670ac20), UINT64_C(0x5b1e309b16452559), - UINT64_C(0xd476a1c3461bbed2), UINT64_C(0xaeaed10b762e37ab), - UINT64_C(0x37deb6af5e9b8b5b), UINT64_C(0x4d06c6676eae0222), - UINT64_C(0xc26e573f3ef099a9), UINT64_C(0xb8b627f70ec510d0), - UINT64_C(0xf7e653dcc6da3dd4), UINT64_C(0x8d3e2314f6efb4ad), - UINT64_C(0x0256b24ca6b12f26), UINT64_C(0x788ec2849684a65f), - UINT64_C(0x9cf65a1b368f752e), UINT64_C(0xe62e2ad306bafc57), - UINT64_C(0x6946bb8b56e467dc), UINT64_C(0x139ecb4366d1eea5), - UINT64_C(0x5ccebf68aecec3a1), UINT64_C(0x2616cfa09efb4ad8), - UINT64_C(0xa97e5ef8cea5d153), UINT64_C(0xd3a62e30fe90582a), - UINT64_C(0xb0c7b7e3c7593bd8), UINT64_C(0xca1fc72bf76cb2a1), - UINT64_C(0x45775673a732292a), UINT64_C(0x3faf26bb9707a053), - UINT64_C(0x70ff52905f188d57), UINT64_C(0x0a2722586f2d042e), - UINT64_C(0x854fb3003f739fa5), UINT64_C(0xff97c3c80f4616dc), - UINT64_C(0x1bef5b57af4dc5ad), UINT64_C(0x61372b9f9f784cd4), - UINT64_C(0xee5fbac7cf26d75f), UINT64_C(0x9487ca0fff135e26), - UINT64_C(0xdbd7be24370c7322), UINT64_C(0xa10fceec0739fa5b), - UINT64_C(0x2e675fb4576761d0), UINT64_C(0x54bf2f7c6752e8a9), - UINT64_C(0xcdcf48d84fe75459), UINT64_C(0xb71738107fd2dd20), - UINT64_C(0x387fa9482f8c46ab), UINT64_C(0x42a7d9801fb9cfd2), - UINT64_C(0x0df7adabd7a6e2d6), UINT64_C(0x772fdd63e7936baf), - UINT64_C(0xf8474c3bb7cdf024), UINT64_C(0x829f3cf387f8795d), - UINT64_C(0x66e7a46c27f3aa2c), UINT64_C(0x1c3fd4a417c62355), - UINT64_C(0x935745fc4798b8de), UINT64_C(0xe98f353477ad31a7), - UINT64_C(0xa6df411fbfb21ca3), UINT64_C(0xdc0731d78f8795da), - UINT64_C(0x536fa08fdfd90e51), UINT64_C(0x29b7d047efec8728), -}; +/******************** BEGIN GENERATED PYCRC FUNCTIONS ********************/ +/** + * Generated on Sun Dec 21 14:14:07 2014, + * by pycrc v0.8.2, https://www.tty1.net/pycrc/ + * + * LICENSE ON GENERATED CODE: + * ========================== + * As of version 0.6, pycrc is released under the terms of the MIT licence. + * The code generated by pycrc is not considered a substantial portion of the + * software, therefore the author of pycrc will not claim any copyright on + * the generated code. + * ========================== + * + * CRC configuration: + * Width = 64 + * Poly = 0xad93d23594c935a9 + * XorIn = 0xffffffffffffffff + * ReflectIn = True + * XorOut = 0x0000000000000000 + * ReflectOut = True + * Algorithm = bit-by-bit-fast + * + * Modifications after generation (by matt): + * - included finalize step in-line with update for single-call generation + * - re-worked some inner variable architectures + * - adjusted function parameters to match expected prototypes. + *****************************************************************************/ -uint64_t crc64(uint64_t crc, const unsigned char *s, uint64_t l) { - uint64_t j; +/** + * Reflect all bits of a \a data word of \a data_len bytes. + * + * \param data The data word to be reflected. + * \param data_len The width of \a data expressed in number of bits. + * \return The reflected data. + *****************************************************************************/ +static inline uint_fast64_t crc_reflect(uint_fast64_t data, size_t data_len) { + uint_fast64_t ret = data & 0x01; - for (j = 0; j < l; j++) { - uint8_t byte = s[j]; - crc = crc64_tab[(uint8_t)crc ^ byte] ^ (crc >> 8); + for (size_t i = 1; i < data_len; i++) { + data >>= 1; + ret = (ret << 1) | (data & 0x01); } - return crc; + + return ret; +} + +/** + * Update the crc value with new data. + * + * \param crc The current crc value. + * \param data Pointer to a buffer of \a data_len bytes. + * \param data_len Number of bytes in the \a data buffer. + * \return The updated crc value. + ******************************************************************************/ +uint64_t _crc64(uint_fast64_t crc, const void *in_data, const uint64_t len) { + const uint8_t *data = in_data; + unsigned long long bit; + + for (uint64_t offset = 0; offset < len; offset++) { + uint8_t c = data[offset]; + for (uint_fast8_t i = 0x01; i & 0xff; i <<= 1) { + bit = crc & 0x8000000000000000; + if (c & i) { + bit = !bit; + } + + crc <<= 1; + if (bit) { + crc ^= POLY; + } + } + + crc &= 0xffffffffffffffff; + } + + crc = crc & 0xffffffffffffffff; + return crc_reflect(crc, 64) ^ 0x0000000000000000; +} + +/******************** END GENERATED PYCRC FUNCTIONS ********************/ + +/* Initializes the 16KB lookup tables. */ +void crc64_init(void) { + crcspeed64native_init(_crc64, crc64_table); +} + +/* Compute crc64 */ +uint64_t crc64(uint64_t crc, const unsigned char *s, uint64_t l) { + return crcspeed64native(crc64_table, crc, (void *) s, l); } /* Test main */ -#ifdef REDIS_TEST +#if defined(REDIS_TEST) || defined(REDIS_TEST_MAIN) #include #define UNUSED(x) (void)(x) int crc64Test(int argc, char *argv[]) { UNUSED(argc); UNUSED(argv); - printf("e9c6d914c4b8d9ca == %016llx\n", - (unsigned long long) crc64(0,(unsigned char*)"123456789",9)); + crc64_init(); + printf("[calcula]: e9c6d914c4b8d9ca == %016" PRIx64 "\n", + (uint64_t)_crc64(0, "123456789", 9)); + printf("[64speed]: e9c6d914c4b8d9ca == %016" PRIx64 "\n", + (uint64_t)crc64speed(0, "123456789", 9)); + char li[] = "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed " + "do eiusmod tempor incididunt ut labore et dolore magna " + "aliqua. Ut enim ad minim veniam, quis nostrud exercitation " + "ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis " + "aute irure dolor in reprehenderit in voluptate velit esse " + "cillum dolore eu fugiat nulla pariatur. Excepteur sint " + "occaecat cupidatat non proident, sunt in culpa qui officia " + "deserunt mollit anim id est laborum."; + printf("[calcula]: c7794709e69683b3 == %016" PRIx64 "\n", + (uint64_t)_crc64(0, li, sizeof(li))); + printf("[64speed]: c7794709e69683b3 == %016" PRIx64 "\n", + (uint64_t)crc64(0, li, sizeof(li))); return 0; } + +#endif + +#ifdef REDIS_TEST_MAIN +int main(int argc, char *argv[]) { + return crc64Test(argc, argv); +} + #endif diff --git a/src/crc64.h b/src/crc64.h index c9fca519d..60c42345f 100644 --- a/src/crc64.h +++ b/src/crc64.h @@ -3,6 +3,7 @@ #include +void crc64_init(void); uint64_t crc64(uint64_t crc, const unsigned char *s, uint64_t l); #ifdef REDIS_TEST diff --git a/src/rdb.c b/src/rdb.c index 143b6c325..9f6bf13f1 100644 --- a/src/rdb.c +++ b/src/rdb.c @@ -2291,7 +2291,8 @@ int rdbLoadRio(rio *rdb, int rdbflags, rdbSaveInfo *rsi) { if (cksum == 0) { serverLog(LL_WARNING,"RDB file was saved with checksum disabled: no check performed."); } else if (cksum != expected) { - serverLog(LL_WARNING,"Wrong RDB checksum. Aborting now."); + serverLog(LL_WARNING,"Wrong RDB checksum expected: (%llx) but " + "got (%llx). Aborting now.",expected,cksum); rdbExitReportCorruptRDB("RDB CRC error"); } } diff --git a/src/server.c b/src/server.c index f2fe54e1c..7835a261a 100644 --- a/src/server.c +++ b/src/server.c @@ -2892,6 +2892,7 @@ void initServer(void) { scriptingInit(1); slowlogInit(); latencyMonitorInit(); + crc64_init(); } /* Some steps in server initialization need to be done last (after modules From 676e31201a5f1caa37c664d3df95dedcac31aa9c Mon Sep 17 00:00:00 2001 From: Madelyn Olson Date: Fri, 24 Apr 2020 17:05:37 -0700 Subject: [PATCH 0310/1280] Made crc64 test consistent --- src/crc64.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/crc64.c b/src/crc64.c index 165941dea..4cbc019f6 100644 --- a/src/crc64.c +++ b/src/crc64.c @@ -30,6 +30,7 @@ #include "crcspeed.h" static uint64_t crc64_table[8][256] = {{0}}; +#define POLY UINT64_C(0xad93d23594c935a9) /******************** BEGIN GENERATED PYCRC FUNCTIONS ********************/ /** * Generated on Sun Dec 21 14:14:07 2014, @@ -122,7 +123,7 @@ uint64_t crc64(uint64_t crc, const unsigned char *s, uint64_t l) { } /* Test main */ -#if defined(REDIS_TEST) || defined(REDIS_TEST_MAIN) +#ifdef REDIS_TEST #include #define UNUSED(x) (void)(x) @@ -133,7 +134,7 @@ int crc64Test(int argc, char *argv[]) { printf("[calcula]: e9c6d914c4b8d9ca == %016" PRIx64 "\n", (uint64_t)_crc64(0, "123456789", 9)); printf("[64speed]: e9c6d914c4b8d9ca == %016" PRIx64 "\n", - (uint64_t)crc64speed(0, "123456789", 9)); + (uint64_t)crc64(0, "123456789", 9)); char li[] = "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed " "do eiusmod tempor incididunt ut labore et dolore magna " "aliqua. Ut enim ad minim veniam, quis nostrud exercitation " From a69f237f77cdaa98cd0d955dd76f8c6dc472936b Mon Sep 17 00:00:00 2001 From: Madelyn Olson Date: Fri, 24 Apr 2020 17:11:21 -0700 Subject: [PATCH 0311/1280] Added crcspeed library --- src/crcspeed.c | 281 +++++++++++++++++++++++++++++++++++++++++++++++++ src/crcspeed.h | 60 +++++++++++ 2 files changed, 341 insertions(+) create mode 100644 src/crcspeed.c create mode 100644 src/crcspeed.h diff --git a/src/crcspeed.c b/src/crcspeed.c new file mode 100644 index 000000000..d2d97a8c7 --- /dev/null +++ b/src/crcspeed.c @@ -0,0 +1,281 @@ +/* + * Copyright (C) 2013 Mark Adler + * Originally by: crc64.c Version 1.4 16 Dec 2013 Mark Adler + * Modifications by Matt Stancliff : + * - removed CRC64-specific behavior + * - added generation of lookup tables by parameters + * - removed inversion of CRC input/result + * - removed automatic initialization in favor of explicit initialization + + This software is provided 'as-is', without any express or implied + warranty. In no event will the author be held liable for any damages + arising from the use of this software. + + Permission is granted to anyone to use this software for any purpose, + including commercial applications, and to alter it and redistribute it + freely, subject to the following restrictions: + + 1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. + 2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. + 3. This notice may not be removed or altered from any source distribution. + + Mark Adler + madler@alumni.caltech.edu + */ + +#include "crcspeed.h" + +/* Fill in a CRC constants table. */ +void crcspeed64little_init(crcfn64 crcfn, uint64_t table[8][256]) { + uint64_t crc; + + /* generate CRCs for all single byte sequences */ + for (int n = 0; n < 256; n++) { + table[0][n] = crcfn(0, &n, 1); + } + + /* generate nested CRC table for future slice-by-8 lookup */ + for (int n = 0; n < 256; n++) { + crc = table[0][n]; + for (int k = 1; k < 8; k++) { + crc = table[0][crc & 0xff] ^ (crc >> 8); + table[k][n] = crc; + } + } +} + +void crcspeed16little_init(crcfn16 crcfn, uint16_t table[8][256]) { + uint16_t crc; + + /* generate CRCs for all single byte sequences */ + for (int n = 0; n < 256; n++) { + table[0][n] = crcfn(0, &n, 1); + } + + /* generate nested CRC table for future slice-by-8 lookup */ + for (int n = 0; n < 256; n++) { + crc = table[0][n]; + for (int k = 1; k < 8; k++) { + crc = table[0][(crc >> 8) & 0xff] ^ (crc << 8); + table[k][n] = crc; + } + } +} + +/* Reverse the bytes in a 64-bit word. */ +static inline uint64_t rev8(uint64_t a) { +#if defined(__GNUC__) || defined(__clang__) + return __builtin_bswap64(a); +#else + uint64_t m; + + m = UINT64_C(0xff00ff00ff00ff); + a = ((a >> 8) & m) | (a & m) << 8; + m = UINT64_C(0xffff0000ffff); + a = ((a >> 16) & m) | (a & m) << 16; + return a >> 32 | a << 32; +#endif +} + +/* This function is called once to initialize the CRC table for use on a + big-endian architecture. */ +void crcspeed64big_init(crcfn64 fn, uint64_t big_table[8][256]) { + /* Create the little endian table then reverse all the entires. */ + crcspeed64little_init(fn, big_table); + for (int k = 0; k < 8; k++) { + for (int n = 0; n < 256; n++) { + big_table[k][n] = rev8(big_table[k][n]); + } + } +} + +void crcspeed16big_init(crcfn16 fn, uint16_t big_table[8][256]) { + /* Create the little endian table then reverse all the entires. */ + crcspeed16little_init(fn, big_table); + for (int k = 0; k < 8; k++) { + for (int n = 0; n < 256; n++) { + big_table[k][n] = rev8(big_table[k][n]); + } + } +} + +/* Calculate a non-inverted CRC multiple bytes at a time on a little-endian + * architecture. If you need inverted CRC, invert *before* calling and invert + * *after* calling. + * 64 bit crc = process 8 bytes at once; + */ +uint64_t crcspeed64little(uint64_t little_table[8][256], uint64_t crc, + void *buf, size_t len) { + unsigned char *next = buf; + + /* process individual bytes until we reach an 8-byte aligned pointer */ + while (len && ((uintptr_t)next & 7) != 0) { + crc = little_table[0][(crc ^ *next++) & 0xff] ^ (crc >> 8); + len--; + } + + /* fast middle processing, 8 bytes (aligned!) per loop */ + while (len >= 8) { + crc ^= *(uint64_t *)next; + crc = little_table[7][crc & 0xff] ^ + little_table[6][(crc >> 8) & 0xff] ^ + little_table[5][(crc >> 16) & 0xff] ^ + little_table[4][(crc >> 24) & 0xff] ^ + little_table[3][(crc >> 32) & 0xff] ^ + little_table[2][(crc >> 40) & 0xff] ^ + little_table[1][(crc >> 48) & 0xff] ^ + little_table[0][crc >> 56]; + next += 8; + len -= 8; + } + + /* process remaining bytes (can't be larger than 8) */ + while (len) { + crc = little_table[0][(crc ^ *next++) & 0xff] ^ (crc >> 8); + len--; + } + + return crc; +} + +uint16_t crcspeed16little(uint16_t little_table[8][256], uint16_t crc, + void *buf, size_t len) { + unsigned char *next = buf; + + /* process individual bytes until we reach an 8-byte aligned pointer */ + while (len && ((uintptr_t)next & 7) != 0) { + crc = little_table[0][((crc >> 8) ^ *next++) & 0xff] ^ (crc << 8); + len--; + } + + /* fast middle processing, 8 bytes (aligned!) per loop */ + while (len >= 8) { + uint64_t n = *(uint64_t *)next; + crc = little_table[7][(n & 0xff) ^ ((crc >> 8) & 0xff)] ^ + little_table[6][((n >> 8) & 0xff) ^ (crc & 0xff)] ^ + little_table[5][(n >> 16) & 0xff] ^ + little_table[4][(n >> 24) & 0xff] ^ + little_table[3][(n >> 32) & 0xff] ^ + little_table[2][(n >> 40) & 0xff] ^ + little_table[1][(n >> 48) & 0xff] ^ + little_table[0][n >> 56]; + next += 8; + len -= 8; + } + + /* process remaining bytes (can't be larger than 8) */ + while (len) { + crc = little_table[0][((crc >> 8) ^ *next++) & 0xff] ^ (crc << 8); + len--; + } + + return crc; +} + +/* Calculate a non-inverted CRC eight bytes at a time on a big-endian + * architecture. + */ +uint64_t crcspeed64big(uint64_t big_table[8][256], uint64_t crc, void *buf, + size_t len) { + unsigned char *next = buf; + + crc = rev8(crc); + while (len && ((uintptr_t)next & 7) != 0) { + crc = big_table[0][(crc >> 56) ^ *next++] ^ (crc << 8); + len--; + } + + while (len >= 8) { + crc ^= *(uint64_t *)next; + crc = big_table[0][crc & 0xff] ^ + big_table[1][(crc >> 8) & 0xff] ^ + big_table[2][(crc >> 16) & 0xff] ^ + big_table[3][(crc >> 24) & 0xff] ^ + big_table[4][(crc >> 32) & 0xff] ^ + big_table[5][(crc >> 40) & 0xff] ^ + big_table[6][(crc >> 48) & 0xff] ^ + big_table[7][crc >> 56]; + next += 8; + len -= 8; + } + + while (len) { + crc = big_table[0][(crc >> 56) ^ *next++] ^ (crc << 8); + len--; + } + + return rev8(crc); +} + +/* WARNING: Completely untested on big endian architecture. Possibly broken. */ +uint16_t crcspeed16big(uint16_t big_table[8][256], uint16_t crc_in, void *buf, + size_t len) { + unsigned char *next = buf; + uint64_t crc = crc_in; + + crc = rev8(crc); + while (len && ((uintptr_t)next & 7) != 0) { + crc = big_table[0][((crc >> (56 - 8)) ^ *next++) & 0xff] ^ (crc >> 8); + len--; + } + + while (len >= 8) { + uint64_t n = *(uint64_t *)next; + crc = big_table[0][(n & 0xff) ^ ((crc >> (56 - 8)) & 0xff)] ^ + big_table[1][((n >> 8) & 0xff) ^ (crc & 0xff)] ^ + big_table[2][(n >> 16) & 0xff] ^ + big_table[3][(n >> 24) & 0xff] ^ + big_table[4][(n >> 32) & 0xff] ^ + big_table[5][(n >> 40) & 0xff] ^ + big_table[6][(n >> 48) & 0xff] ^ + big_table[7][n >> 56]; + next += 8; + len -= 8; + } + + while (len) { + crc = big_table[0][((crc >> (56 - 8)) ^ *next++) & 0xff] ^ (crc >> 8); + len--; + } + + return rev8(crc); +} + +/* Return the CRC of buf[0..len-1] with initial crc, processing eight bytes + at a time using passed-in lookup table. + This selects one of two routines depending on the endianess of + the architecture. */ +uint64_t crcspeed64native(uint64_t table[8][256], uint64_t crc, void *buf, + size_t len) { + uint64_t n = 1; + + return *(char *)&n ? crcspeed64little(table, crc, buf, len) + : crcspeed64big(table, crc, buf, len); +} + +uint16_t crcspeed16native(uint16_t table[8][256], uint16_t crc, void *buf, + size_t len) { + uint64_t n = 1; + + return *(char *)&n ? crcspeed16little(table, crc, buf, len) + : crcspeed16big(table, crc, buf, len); +} + +/* Initialize CRC lookup table in architecture-dependent manner. */ +void crcspeed64native_init(crcfn64 fn, uint64_t table[8][256]) { + uint64_t n = 1; + + *(char *)&n ? crcspeed64little_init(fn, table) + : crcspeed64big_init(fn, table); +} + +void crcspeed16native_init(crcfn16 fn, uint16_t table[8][256]) { + uint64_t n = 1; + + *(char *)&n ? crcspeed16little_init(fn, table) + : crcspeed16big_init(fn, table); +} diff --git a/src/crcspeed.h b/src/crcspeed.h new file mode 100644 index 000000000..d7ee95ebb --- /dev/null +++ b/src/crcspeed.h @@ -0,0 +1,60 @@ +/* Copyright (c) 2014, Matt Stancliff + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. */ + +#ifndef CRCSPEED_H +#define CRCSPEED_H + +#include +#include + +typedef uint64_t (*crcfn64)(uint64_t, const void *, const uint64_t); +typedef uint16_t (*crcfn16)(uint16_t, const void *, const uint64_t); + +/* CRC-64 */ +void crcspeed64little_init(crcfn64 fn, uint64_t table[8][256]); +void crcspeed64big_init(crcfn64 fn, uint64_t table[8][256]); +void crcspeed64native_init(crcfn64 fn, uint64_t table[8][256]); + +uint64_t crcspeed64little(uint64_t table[8][256], uint64_t crc, void *buf, + size_t len); +uint64_t crcspeed64big(uint64_t table[8][256], uint64_t crc, void *buf, + size_t len); +uint64_t crcspeed64native(uint64_t table[8][256], uint64_t crc, void *buf, + size_t len); + +/* CRC-16 */ +void crcspeed16little_init(crcfn16 fn, uint16_t table[8][256]); +void crcspeed16big_init(crcfn16 fn, uint16_t table[8][256]); +void crcspeed16native_init(crcfn16 fn, uint16_t table[8][256]); + +uint16_t crcspeed16little(uint16_t table[8][256], uint16_t crc, void *buf, + size_t len); +uint16_t crcspeed16big(uint16_t table[8][256], uint16_t crc, void *buf, + size_t len); +uint16_t crcspeed16native(uint16_t table[8][256], uint16_t crc, void *buf, + size_t len); +#endif From 23a3f0bd14680c896018ae6be4c112b85a2875ad Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 27 Apr 2020 13:35:17 +0200 Subject: [PATCH 0312/1280] Fix STRALGO command flags. --- src/server.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/server.c b/src/server.c index f2fe54e1c..db1d3780c 100644 --- a/src/server.c +++ b/src/server.c @@ -1007,7 +1007,7 @@ struct redisCommand redisCommandTable[] = { 0,NULL,0,0,0,0,0,0}, {"stralgo",stralgoCommand,-2, - "write use-memory @string", + "read-only @string", 0,lcsGetKeys,0,0,0,0,0,0} }; From 56338629248dc9527411f32a88815423d98778d8 Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Thu, 23 Apr 2020 15:04:42 +0300 Subject: [PATCH 0313/1280] Keep track of meaningful replication offset in replicas too Now both master and replicas keep track of the last replication offset that contains meaningful data (ignoring the tailing pings), and both trim that tail from the replication backlog, and the offset with which they try to use for psync. the implication is that if someone missed some pings, or even have excessive pings that the promoted replica has, it'll still be able to psync (avoid full sync). the downside (which was already committed) is that replicas running old code may fail to psync, since the promoted replica trims pings form it's backlog. This commit adds a test that reproduces several cases of promotions and demotions with stale and non-stale pings Background: The mearningful offset on the master was added recently to solve a problem were the master is left all alone, injecting PINGs into it's backlog when no one is listening and then gets demoted and tries to replicate from a replica that didn't have any of the PINGs (or at least not the last ones). however, consider this case: master A has two replicas (B and C) replicating directly from it. there's no traffic at all, and also no network issues, just many pings in the tail of the backlog. now B gets promoted, A becomes a replica of B, and C remains a replica of A. when A gets demoted, it trims the pings from its backlog, and successfully replicate from B. however, C is still aware of these PINGs, when it'll disconnect and re-connect to A, it'll ask for something that's not in the backlog anymore (since A trimmed the tail of it's backlog), and be forced to do a full sync (something it didn't have to do before the meaningful offset fix). Besides that, the psync2 test was always failing randomly here and there, it turns out the reason were PINGs. Investigating it shows the following scenario: cycle 1: redis #1 is master, and all the rest are direct replicas of #1 cycle 2: redis #2 is promoted to master, #1 is a replica of #2 and #3 is replica of #1 now we see that when #1 is demoted it prints: 17339:S 21 Apr 2020 11:16:38.523 * Using the meaningful offset 3929963 instead of 3929977 to exclude the final PINGs (14 bytes difference) 17339:S 21 Apr 2020 11:16:39.391 * Trying a partial resynchronization (request e2b3f8817735fdfe5fa4626766daa938b61419e5:3929964). 17339:S 21 Apr 2020 11:16:39.392 * Successful partial resynchronization with master. and when #3 connects to the demoted #2, #2 says: 17339:S 21 Apr 2020 11:16:40.084 * Partial resynchronization not accepted: Requested offset for secondary ID was 3929978, but I can reply up to 3929964 so the issue here is that the meaningful offset feature saved the day for the demoted master (since it needs to sync from a replica that didn't get the last ping), but it didn't help one of the other replicas which did get the last ping. --- src/blocked.c | 2 +- src/networking.c | 93 +++++++++++----------- src/replication.c | 66 +++++++++------- src/server.h | 1 - tests/integration/psync2.tcl | 144 ++++++++++++++++++++++++++++++----- 5 files changed, 213 insertions(+), 93 deletions(-) diff --git a/src/blocked.c b/src/blocked.c index 00cc798d5..045369e93 100644 --- a/src/blocked.c +++ b/src/blocked.c @@ -110,7 +110,7 @@ void processUnblockedClients(void) { * the code is conceptually more correct this way. */ if (!(c->flags & CLIENT_BLOCKED)) { if (c->querybuf && sdslen(c->querybuf) > 0) { - processInputBufferAndReplicate(c); + processInputBuffer(c); } } } diff --git a/src/networking.c b/src/networking.c index 744979d16..e4d40fdf0 100644 --- a/src/networking.c +++ b/src/networking.c @@ -1671,12 +1671,55 @@ int processMultibulkBuffer(client *c) { return C_ERR; } +/* Perform necessary tasks after a command was executed: + * + * 1. The client is reset unless there are reasons to avoid doing it. + * 2. In the case of master clients, the replication offset is updated. + * 3. Propagate commands we got from our master to replicas down the line. */ +void commandProcessed(client *c) { + int cmd_is_ping = c->cmd && c->cmd->proc == pingCommand; + long long prev_offset = c->reploff; + if (c->flags & CLIENT_MASTER && !(c->flags & CLIENT_MULTI)) { + /* Update the applied replication offset of our master. */ + c->reploff = c->read_reploff - sdslen(c->querybuf) + c->qb_pos; + } + + /* Don't reset the client structure for clients blocked in a + * module blocking command, so that the reply callback will + * still be able to access the client argv and argc field. + * The client will be reset in unblockClientFromModule(). */ + if (!(c->flags & CLIENT_BLOCKED) || + c->btype != BLOCKED_MODULE) + { + resetClient(c); + } + + /* If the client is a master we need to compute the difference + * between the applied offset before and after processing the buffer, + * to understand how much of the replication stream was actually + * applied to the master state: this quantity, and its corresponding + * part of the replication stream, will be propagated to the + * sub-replicas and to the replication backlog. */ + if (c->flags & CLIENT_MASTER) { + long long applied = c->reploff - prev_offset; + long long prev_master_repl_meaningful_offset = server.master_repl_meaningful_offset; + if (applied) { + replicationFeedSlavesFromMasterStream(server.slaves, + c->pending_querybuf, applied); + sdsrange(c->pending_querybuf,applied,-1); + } + /* The server.master_repl_meaningful_offset variable represents + * the offset of the replication stream without the pending PINGs. */ + if (cmd_is_ping) + server.master_repl_meaningful_offset = prev_master_repl_meaningful_offset; + } +} + /* This function calls processCommand(), but also performs a few sub tasks - * that are useful in that context: + * for the client that are useful in that context: * * 1. It sets the current client to the client 'c'. - * 2. In the case of master clients, the replication offset is updated. - * 3. The client is reset unless there are reasons to avoid doing it. + * 2. calls commandProcessed() if the command was handled. * * The function returns C_ERR in case the client was freed as a side effect * of processing the command, otherwise C_OK is returned. */ @@ -1684,20 +1727,7 @@ int processCommandAndResetClient(client *c) { int deadclient = 0; server.current_client = c; if (processCommand(c) == C_OK) { - if (c->flags & CLIENT_MASTER && !(c->flags & CLIENT_MULTI)) { - /* Update the applied replication offset of our master. */ - c->reploff = c->read_reploff - sdslen(c->querybuf) + c->qb_pos; - } - - /* Don't reset the client structure for clients blocked in a - * module blocking command, so that the reply callback will - * still be able to access the client argv and argc field. - * The client will be reset in unblockClientFromModule(). */ - if (!(c->flags & CLIENT_BLOCKED) || - c->btype != BLOCKED_MODULE) - { - resetClient(c); - } + commandProcessed(c); } if (server.current_client == NULL) deadclient = 1; server.current_client = NULL; @@ -1794,31 +1824,6 @@ void processInputBuffer(client *c) { } } -/* This is a wrapper for processInputBuffer that also cares about handling - * the replication forwarding to the sub-replicas, in case the client 'c' - * is flagged as master. Usually you want to call this instead of the - * raw processInputBuffer(). */ -void processInputBufferAndReplicate(client *c) { - if (!(c->flags & CLIENT_MASTER)) { - processInputBuffer(c); - } else { - /* If the client is a master we need to compute the difference - * between the applied offset before and after processing the buffer, - * to understand how much of the replication stream was actually - * applied to the master state: this quantity, and its corresponding - * part of the replication stream, will be propagated to the - * sub-replicas and to the replication backlog. */ - size_t prev_offset = c->reploff; - processInputBuffer(c); - size_t applied = c->reploff - prev_offset; - if (applied) { - replicationFeedSlavesFromMasterStream(server.slaves, - c->pending_querybuf, applied); - sdsrange(c->pending_querybuf,applied,-1); - } - } -} - void readQueryFromClient(connection *conn) { client *c = connGetPrivateData(conn); int nread, readlen; @@ -1886,7 +1891,7 @@ void readQueryFromClient(connection *conn) { /* There is more data in the client input buffer, continue parsing it * in case to check if there is a full command to execute. */ - processInputBufferAndReplicate(c); + processInputBuffer(c); } void getClientsMaxBuffers(unsigned long *longest_output_list, @@ -3101,7 +3106,7 @@ int handleClientsWithPendingReadsUsingThreads(void) { continue; } } - processInputBufferAndReplicate(c); + processInputBuffer(c); } return processed; } diff --git a/src/replication.c b/src/replication.c index 3e9910374..c59639cd1 100644 --- a/src/replication.c +++ b/src/replication.c @@ -39,6 +39,7 @@ #include #include +long long adjustMeaningfulReplOffset(); void replicationDiscardCachedMaster(void); void replicationResurrectCachedMaster(connection *conn); void replicationSendAck(void); @@ -2693,6 +2694,9 @@ void replicationCacheMaster(client *c) { * pending outputs to the master. */ sdsclear(server.master->querybuf); sdsclear(server.master->pending_querybuf); + /* Adjust reploff and read_reploff to the last meaningful offset we executed. + * this is the offset the replica will use for future PSYNC. */ + server.master->reploff = adjustMeaningfulReplOffset(); server.master->read_reploff = server.master->reploff; if (c->flags & CLIENT_MULTI) discardTransaction(c); listEmpty(c->reply); @@ -2717,6 +2721,38 @@ void replicationCacheMaster(client *c) { replicationHandleMasterDisconnection(); } +/* If the "meaningful" offset, that is the offset without the final PINGs + * in the stream, is different than the last offset, use it instead: + * often when the master is no longer reachable, replicas will never + * receive the PINGs, however the master will end with an incremented + * offset because of the PINGs and will not be able to incrementally + * PSYNC with the new master. + * This function trims the replication backlog when needed, and returns + * the offset to be used for future partial sync. */ +long long adjustMeaningfulReplOffset() { + if (server.master_repl_offset > server.master_repl_meaningful_offset) { + long long delta = server.master_repl_offset - + server.master_repl_meaningful_offset; + serverLog(LL_NOTICE, + "Using the meaningful offset %lld instead of %lld to exclude " + "the final PINGs (%lld bytes difference)", + server.master_repl_meaningful_offset, + server.master_repl_offset, + delta); + server.master_repl_offset = server.master_repl_meaningful_offset; + if (server.repl_backlog_histlen <= delta) { + server.repl_backlog_histlen = 0; + server.repl_backlog_idx = 0; + } else { + server.repl_backlog_histlen -= delta; + server.repl_backlog_idx = + (server.repl_backlog_idx + (server.repl_backlog_size - delta)) % + server.repl_backlog_size; + } + } + return server.master_repl_offset; +} + /* This function is called when a master is turend into a slave, in order to * create from scratch a cached master for the new client, that will allow * to PSYNC with the slave that was promoted as the new master after a @@ -2736,35 +2772,7 @@ void replicationCacheMasterUsingMyself(void) { * by replicationCreateMasterClient(). We'll later set the created * master as server.cached_master, so the replica will use such * offset for PSYNC. */ - server.master_initial_offset = server.master_repl_offset; - - /* However if the "meaningful" offset, that is the offset without - * the final PINGs in the stream, is different, use this instead: - * often when the master is no longer reachable, replicas will never - * receive the PINGs, however the master will end with an incremented - * offset because of the PINGs and will not be able to incrementally - * PSYNC with the new master. */ - if (server.master_repl_offset > server.master_repl_meaningful_offset) { - long long delta = server.master_repl_offset - - server.master_repl_meaningful_offset; - serverLog(LL_NOTICE, - "Using the meaningful offset %lld instead of %lld to exclude " - "the final PINGs (%lld bytes difference)", - server.master_repl_meaningful_offset, - server.master_repl_offset, - delta); - server.master_initial_offset = server.master_repl_meaningful_offset; - server.master_repl_offset = server.master_repl_meaningful_offset; - if (server.repl_backlog_histlen <= delta) { - server.repl_backlog_histlen = 0; - server.repl_backlog_idx = 0; - } else { - server.repl_backlog_histlen -= delta; - server.repl_backlog_idx = - (server.repl_backlog_idx + (server.repl_backlog_size - delta)) % - server.repl_backlog_size; - } - } + server.master_initial_offset = adjustMeaningfulReplOffset(); /* The master client we create can be set to any DBID, because * the new master will start its replication stream with SELECT. */ diff --git a/src/server.h b/src/server.h index 9e1e506af..41d767e13 100644 --- a/src/server.h +++ b/src/server.h @@ -1600,7 +1600,6 @@ void setDeferredSetLen(client *c, void *node, long length); void setDeferredAttributeLen(client *c, void *node, long length); void setDeferredPushLen(client *c, void *node, long length); void processInputBuffer(client *c); -void processInputBufferAndReplicate(client *c); void processGopherRequest(client *c); void acceptHandler(aeEventLoop *el, int fd, void *privdata, int mask); void acceptTcpHandler(aeEventLoop *el, int fd, void *privdata, int mask); diff --git a/tests/integration/psync2.tcl b/tests/integration/psync2.tcl index 333736ffa..4e1189e0b 100644 --- a/tests/integration/psync2.tcl +++ b/tests/integration/psync2.tcl @@ -44,6 +44,7 @@ start_server {} { set used [list $master_id] test "PSYNC2: \[NEW LAYOUT\] Set #$master_id as master" { $R($master_id) slaveof no one + $R($master_id) config set repl-ping-replica-period 1 ;# increse the chance that random ping will cause issues if {$counter_value == 0} { $R($master_id) set x $counter_value } @@ -114,23 +115,20 @@ start_server {} { } } - # wait for all the slaves to be in sync with the master - set master_ofs [status $R($master_id) master_repl_offset] + # wait for all the slaves to be in sync with the master, due to pings, we have to re-sample the master constantly too wait_for_condition 500 100 { - $master_ofs == [status $R(0) master_repl_offset] && - $master_ofs == [status $R(1) master_repl_offset] && - $master_ofs == [status $R(2) master_repl_offset] && - $master_ofs == [status $R(3) master_repl_offset] && - $master_ofs == [status $R(4) master_repl_offset] + [status $R($master_id) master_repl_offset] == [status $R(0) master_repl_offset] && + [status $R($master_id) master_repl_offset] == [status $R(1) master_repl_offset] && + [status $R($master_id) master_repl_offset] == [status $R(2) master_repl_offset] && + [status $R($master_id) master_repl_offset] == [status $R(3) master_repl_offset] && + [status $R($master_id) master_repl_offset] == [status $R(4) master_repl_offset] } else { - if {$debug_msg} { - for {set j 0} {$j < 5} {incr j} { - puts "$j: sync_full: [status $R($j) sync_full]" - puts "$j: id1 : [status $R($j) master_replid]:[status $R($j) master_repl_offset]" - puts "$j: id2 : [status $R($j) master_replid2]:[status $R($j) second_repl_offset]" - puts "$j: backlog : firstbyte=[status $R($j) repl_backlog_first_byte_offset] len=[status $R($j) repl_backlog_histlen]" - puts "---" - } + for {set j 0} {$j < 5} {incr j} { + puts "$j: sync_full: [status $R($j) sync_full]" + puts "$j: id1 : [status $R($j) master_replid]:[status $R($j) master_repl_offset]" + puts "$j: id2 : [status $R($j) master_replid2]:[status $R($j) second_repl_offset]" + puts "$j: backlog : firstbyte=[status $R($j) repl_backlog_first_byte_offset] len=[status $R($j) repl_backlog_histlen]" + puts "---" } fail "Slaves are not in sync with the master after too long time." } @@ -175,9 +173,14 @@ start_server {} { $R($j) slaveof $master_host $master_port } - # Wait for slaves to sync + # Wait for replicas to sync. it is not enough to just wait for connected_slaves==4 + # since we might do the check before the master realized that they're disconnected wait_for_condition 50 1000 { - [status $R($master_id) connected_slaves] == 4 + [status $R($master_id) connected_slaves] == 4 && + [status $R([expr {($master_id+1)%5}]) master_link_status] == "up" && + [status $R([expr {($master_id+2)%5}]) master_link_status] == "up" && + [status $R([expr {($master_id+3)%5}]) master_link_status] == "up" && + [status $R([expr {($master_id+4)%5}]) master_link_status] == "up" } else { fail "Replica not reconnecting" } @@ -188,6 +191,7 @@ start_server {} { set slave_id [expr {($master_id+1)%5}] set sync_count [status $R($master_id) sync_full] set sync_partial [status $R($master_id) sync_partial_ok] + set sync_partial_err [status $R($master_id) sync_partial_err] catch { $R($slave_id) config rewrite $R($slave_id) debug restart @@ -197,7 +201,11 @@ start_server {} { wait_for_condition 50 1000 { [status $R($master_id) sync_partial_ok] == $sync_partial + 1 } else { - fail "Replica not reconnecting" + puts "prev sync_full: $sync_count" + puts "prev sync_partial_ok: $sync_partial" + puts "prev sync_partial_err: $sync_partial_err" + puts [$R($master_id) info stats] + fail "Replica didn't partial sync" } set new_sync_count [status $R($master_id) sync_full] assert {$sync_count == $new_sync_count} @@ -271,3 +279,103 @@ start_server {} { } }}}}} + +start_server {tags {"psync2"}} { +start_server {} { +start_server {} { +start_server {} { +start_server {} { + test {pings at the end of replication stream are ignored for psync} { + set master [srv -4 client] + set master_host [srv -4 host] + set master_port [srv -4 port] + set replica1 [srv -3 client] + set replica2 [srv -2 client] + set replica3 [srv -1 client] + set replica4 [srv -0 client] + + $replica1 replicaof $master_host $master_port + $replica2 replicaof $master_host $master_port + $replica3 replicaof $master_host $master_port + $replica4 replicaof $master_host $master_port + wait_for_condition 50 1000 { + [status $master connected_slaves] == 4 + } else { + fail "replicas didn't connect" + } + + $master incr x + wait_for_condition 50 1000 { + [$replica1 get x] == 1 && [$replica2 get x] == 1 && + [$replica3 get x] == 1 && [$replica4 get x] == 1 + } else { + fail "replicas didn't get incr" + } + + # disconnect replica1 and replica2 + # and wait for the master to send a ping to replica3 and replica4 + $replica1 replicaof no one + $replica2 replicaof 127.0.0.1 1 ;# we can't promote it to master since that will cycle the replication id + $master config set repl-ping-replica-period 1 + after 1500 + + # make everyone sync from the replica1 that didn't get the last ping from the old master + # replica4 will keep syncing from the old master which now syncs from replica1 + # and replica2 will re-connect to the old master (which went back in time) + set new_master_host [srv -3 host] + set new_master_port [srv -3 port] + $replica3 replicaof $new_master_host $new_master_port + $master replicaof $new_master_host $new_master_port + $replica2 replicaof $master_host $master_port + wait_for_condition 50 1000 { + [status $replica2 master_link_status] == "up" && + [status $replica3 master_link_status] == "up" && + [status $replica4 master_link_status] == "up" && + [status $master master_link_status] == "up" + } else { + fail "replicas didn't connect" + } + + # make sure replication is still alive and kicking + $replica1 incr x + wait_for_condition 50 1000 { + [$replica2 get x] == 2 && + [$replica3 get x] == 2 && + [$replica4 get x] == 2 && + [$master get x] == 2 + } else { + fail "replicas didn't get incr" + } + + # make sure there are full syncs other than the initial ones + assert_equal [status $master sync_full] 4 + assert_equal [status $replica1 sync_full] 0 + assert_equal [status $replica2 sync_full] 0 + assert_equal [status $replica3 sync_full] 0 + assert_equal [status $replica4 sync_full] 0 + + # force psync + $master client kill type master + $replica2 client kill type master + $replica3 client kill type master + $replica4 client kill type master + + # make sure replication is still alive and kicking + $replica1 incr x + wait_for_condition 50 1000 { + [$replica2 get x] == 3 && + [$replica3 get x] == 3 && + [$replica4 get x] == 3 && + [$master get x] == 3 + } else { + fail "replicas didn't get incr" + } + + # make sure there are full syncs other than the initial ones + assert_equal [status $master sync_full] 4 + assert_equal [status $replica1 sync_full] 0 + assert_equal [status $replica2 sync_full] 0 + assert_equal [status $replica3 sync_full] 0 + assert_equal [status $replica4 sync_full] 0 +} +}}}}} From 9499903e5657e54e2e39542ebb1057a20b338c69 Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Mon, 27 Apr 2020 23:17:19 +0300 Subject: [PATCH 0314/1280] allow dictFind using static robj since the recent addition of OBJ_STATIC_REFCOUNT and the assertion in incrRefCount it is now impossible to use dictFind using a static robj, because dictEncObjKeyCompare will call getDecodedObject which tries to increment the refcount just in order to decrement it later. --- src/server.c | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/server.c b/src/server.c index e265fe9ab..1bfca1a0b 100644 --- a/src/server.c +++ b/src/server.c @@ -1219,11 +1219,16 @@ int dictEncObjKeyCompare(void *privdata, const void *key1, o2->encoding == OBJ_ENCODING_INT) return o1->ptr == o2->ptr; - o1 = getDecodedObject(o1); - o2 = getDecodedObject(o2); + /* due to OBJ_STATIC_REFCOUNT, we rather not call sdsEncodedObject unnecessarily */ + if (!sdsEncodedObject(o1)) + o1 = getDecodedObject(o1); + if (!sdsEncodedObject(o2)) + o2 = getDecodedObject(o2); cmp = dictSdsKeyCompare(privdata,o1->ptr,o2->ptr); - decrRefCount(o1); - decrRefCount(o2); + if (o1!=key1) + decrRefCount(o1); + if (o2!=key2) + decrRefCount(o2); return cmp; } From bee51ef883b2e328cc4a4f666c4f66da641d0bbb Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 27 Apr 2020 22:40:15 +0200 Subject: [PATCH 0315/1280] Rework comment in dictEncObjKeyCompare(). --- src/server.c | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/src/server.c b/src/server.c index 1bfca1a0b..1086dab48 100644 --- a/src/server.c +++ b/src/server.c @@ -1219,16 +1219,15 @@ int dictEncObjKeyCompare(void *privdata, const void *key1, o2->encoding == OBJ_ENCODING_INT) return o1->ptr == o2->ptr; - /* due to OBJ_STATIC_REFCOUNT, we rather not call sdsEncodedObject unnecessarily */ - if (!sdsEncodedObject(o1)) - o1 = getDecodedObject(o1); - if (!sdsEncodedObject(o2)) - o2 = getDecodedObject(o2); + /* Due to OBJ_STATIC_REFCOUNT, we avoid calling getDecodedObject() without + * good reasons, because it would incrRefCount() the object, which + * is invalid. So we check to make sure dictFind() works with static + * objects as well. */ + if (!sdsEncodedObject(o1)) o1 = getDecodedObject(o1); + if (!sdsEncodedObject(o2)) o2 = getDecodedObject(o2); cmp = dictSdsKeyCompare(privdata,o1->ptr,o2->ptr); - if (o1!=key1) - decrRefCount(o1); - if (o2!=key2) - decrRefCount(o2); + if (o1!=key1) decrRefCount(o1); + if (o2!=key2) decrRefCount(o2); return cmp; } From a29e617381ebebf24f4ef14a53d35d523e980e10 Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Tue, 28 Apr 2020 09:18:01 +0300 Subject: [PATCH 0316/1280] fix loading race in psync2 tests --- tests/integration/psync2-pingoff.tcl | 1 + tests/integration/psync2-reg.tcl | 5 ++++- tests/integration/psync2.tcl | 10 ++++++++++ 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/tests/integration/psync2-pingoff.tcl b/tests/integration/psync2-pingoff.tcl index 1cea290e7..420747d21 100644 --- a/tests/integration/psync2-pingoff.tcl +++ b/tests/integration/psync2-pingoff.tcl @@ -20,6 +20,7 @@ start_server {} { $R(1) replicaof $R_host(0) $R_port(0) $R(0) set foo bar wait_for_condition 50 1000 { + [status $R(1) master_link_status] == "up" && [$R(0) dbsize] == 1 && [$R(1) dbsize] == 1 } else { fail "Replicas not replicating from master" diff --git a/tests/integration/psync2-reg.tcl b/tests/integration/psync2-reg.tcl index b5ad021e2..71a1c0eb2 100644 --- a/tests/integration/psync2-reg.tcl +++ b/tests/integration/psync2-reg.tcl @@ -28,7 +28,10 @@ start_server {} { $R(2) slaveof $R_host(0) $R_port(0) $R(0) set foo bar wait_for_condition 50 1000 { - [$R(1) dbsize] == 1 && [$R(2) dbsize] == 1 + [status $R(1) master_link_status] == "up" && + [status $R(2) master_link_status] == "up" && + [$R(1) dbsize] == 1 && + [$R(2) dbsize] == 1 } else { fail "Replicas not replicating from master" } diff --git a/tests/integration/psync2.tcl b/tests/integration/psync2.tcl index 4e1189e0b..5fe29caba 100644 --- a/tests/integration/psync2.tcl +++ b/tests/integration/psync2.tcl @@ -67,6 +67,16 @@ start_server {} { lappend used $slave_id } + # Wait for replicas to sync. so next loop won't get -LOADING error + wait_for_condition 50 1000 { + [status $R([expr {($master_id+1)%5}]) master_link_status] == "up" && + [status $R([expr {($master_id+2)%5}]) master_link_status] == "up" && + [status $R([expr {($master_id+3)%5}]) master_link_status] == "up" && + [status $R([expr {($master_id+4)%5}]) master_link_status] == "up" + } else { + fail "Replica not reconnecting" + } + # 3) Increment the counter and wait for all the instances # to converge. test "PSYNC2: cluster is consistent after failover" { From 3a8f8fa487e2dbf29d5f4f38ff2e6b74b03a307a Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Tue, 28 Apr 2020 12:14:46 +0300 Subject: [PATCH 0317/1280] hickup, re-fix dictEncObjKeyCompare come to think of it, in theory (not in practice), getDecodedObject can return the same original object with refcount incremented, so the pointer comparision in the previous commit was invalid. so now instead of checking the encoding, we explicitly check the refcount. --- src/server.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/server.c b/src/server.c index 1086dab48..ab6280121 100644 --- a/src/server.c +++ b/src/server.c @@ -1223,11 +1223,11 @@ int dictEncObjKeyCompare(void *privdata, const void *key1, * good reasons, because it would incrRefCount() the object, which * is invalid. So we check to make sure dictFind() works with static * objects as well. */ - if (!sdsEncodedObject(o1)) o1 = getDecodedObject(o1); - if (!sdsEncodedObject(o2)) o2 = getDecodedObject(o2); + if (o1->refcount != OBJ_STATIC_REFCOUNT) o1 = getDecodedObject(o1); + if (o2->refcount != OBJ_STATIC_REFCOUNT) o2 = getDecodedObject(o2); cmp = dictSdsKeyCompare(privdata,o1->ptr,o2->ptr); - if (o1!=key1) decrRefCount(o1); - if (o2!=key2) decrRefCount(o2); + if (o1->refcount != OBJ_STATIC_REFCOUNT) decrRefCount(o1); + if (o2->refcount != OBJ_STATIC_REFCOUNT) decrRefCount(o2); return cmp; } From b83ae071176a2d925810e64ca44b44e177cf0133 Mon Sep 17 00:00:00 2001 From: Guy Benoish Date: Wed, 22 Apr 2020 16:05:57 +0300 Subject: [PATCH 0318/1280] Extend XINFO STREAM output Introducing XINFO STREAM FULL --- src/t_stream.c | 226 ++++++++++++++++++++++++----- tests/unit/type/stream-cgroups.tcl | 34 +++++ 2 files changed, 226 insertions(+), 34 deletions(-) diff --git a/src/t_stream.c b/src/t_stream.c index 155167af9..508c22225 100644 --- a/src/t_stream.c +++ b/src/t_stream.c @@ -2475,16 +2475,200 @@ void xtrimCommand(client *c) { addReplyLongLong(c,deleted); } +/* Helper function for xinfoCommand. + * Handles the variants of XINFO STREAM */ +void xinfoReplyWithStreamInfo(client *c, stream *s) { + int full = 1; + long long count = 0; + robj **optv = c->argv + 3; /* Options start after XINFO STREAM */ + int optc = c->argc - 3; + + /* Parse options. */ + if (optc == 0) { + full = 0; + } else { + /* Valid options are [FULL] or [FULL COUNT ] */ + if (optc != 1 && optc != 3) { + addReplySubcommandSyntaxError(c); + return; + } + + /* First option must be "FULL" */ + if (strcasecmp(optv[0]->ptr,"full")) { + addReplySubcommandSyntaxError(c); + return; + } + + if (optc == 3) { + /* First option must be "FULL" */ + if (strcasecmp(optv[1]->ptr,"count")) { + addReplySubcommandSyntaxError(c); + return; + } + if (getLongLongFromObjectOrReply(c,optv[2],&count,NULL) == C_ERR) + return; + if (count < 0) count = 0; + } + } + + addReplyMapLen(c,full ? 6 : 7); + addReplyBulkCString(c,"length"); + addReplyLongLong(c,s->length); + addReplyBulkCString(c,"radix-tree-keys"); + addReplyLongLong(c,raxSize(s->rax)); + addReplyBulkCString(c,"radix-tree-nodes"); + addReplyLongLong(c,s->rax->numnodes); + addReplyBulkCString(c,"last-generated-id"); + addReplyStreamID(c,&s->last_id); + + if (!full) { + /* XINFO STREAM */ + + addReplyBulkCString(c,"groups"); + addReplyLongLong(c,s->cgroups ? raxSize(s->cgroups) : 0); + + /* To emit the first/last entry we use streamReplyWithRange(). */ + int emitted; + streamID start, end; + start.ms = start.seq = 0; + end.ms = end.seq = UINT64_MAX; + addReplyBulkCString(c,"first-entry"); + emitted = streamReplyWithRange(c,s,&start,&end,1,0,NULL,NULL, + STREAM_RWR_RAWENTRIES,NULL); + if (!emitted) addReplyNull(c); + addReplyBulkCString(c,"last-entry"); + emitted = streamReplyWithRange(c,s,&start,&end,1,1,NULL,NULL, + STREAM_RWR_RAWENTRIES,NULL); + if (!emitted) addReplyNull(c); + } else { + /* XINFO STREAM FULL [COUNT ] */ + + /* Stream entries */ + addReplyBulkCString(c,"entries"); + streamReplyWithRange(c,s,NULL,NULL,count,0,NULL,NULL,0,NULL); + + /* Consumer groups */ + addReplyBulkCString(c,"groups"); + if (s->cgroups == NULL) { + addReplyArrayLen(c,0); + } else { + addReplyArrayLen(c,raxSize(s->cgroups)); + raxIterator ri_cgroups; + raxStart(&ri_cgroups,s->cgroups); + raxSeek(&ri_cgroups,"^",NULL,0); + while(raxNext(&ri_cgroups)) { + streamCG *cg = ri_cgroups.data; + addReplyMapLen(c,5); + + /* Name */ + addReplyBulkCString(c,"name"); + addReplyBulkCBuffer(c,ri_cgroups.key,ri_cgroups.key_len); + + /* Last delivered ID */ + addReplyBulkCString(c,"last-delivered-id"); + addReplyStreamID(c,&cg->last_id); + + /* Group PEL count */ + addReplyBulkCString(c,"pel-count"); + addReplyLongLong(c,raxSize(cg->pel)); + + /* Group PEL */ + addReplyBulkCString(c,"pending"); + long long arraylen_cg_pel = 0; + void *arrayptr_cg_pel = addReplyDeferredLen(c); + raxIterator ri_cg_pel; + raxStart(&ri_cg_pel,cg->pel); + raxSeek(&ri_cg_pel,"^",NULL,0); + while(raxNext(&ri_cg_pel) && (!count || arraylen_cg_pel < count)) { + streamNACK *nack = ri_cg_pel.data; + addReplyArrayLen(c,4); + + /* Entry ID. */ + streamID id; + streamDecodeID(ri_cg_pel.key,&id); + addReplyStreamID(c,&id); + + /* Consumer name. */ + addReplyBulkCBuffer(c,nack->consumer->name, + sdslen(nack->consumer->name)); + + /* Last delivery. */ + addReplyLongLong(c,nack->delivery_time); + + /* Number of deliveries. */ + addReplyLongLong(c,nack->delivery_count); + + arraylen_cg_pel++; + } + setDeferredArrayLen(c,arrayptr_cg_pel,arraylen_cg_pel); + raxStop(&ri_cg_pel); + + /* Consumers */ + addReplyBulkCString(c,"consumers"); + addReplyArrayLen(c,raxSize(cg->consumers)); + raxIterator ri_consumers; + raxStart(&ri_consumers,cg->consumers); + raxSeek(&ri_consumers,"^",NULL,0); + while(raxNext(&ri_consumers)) { + streamConsumer *consumer = ri_consumers.data; + addReplyMapLen(c,4); + + /* Consumer name */ + addReplyBulkCString(c,"name"); + addReplyBulkCBuffer(c,consumer->name,sdslen(consumer->name)); + + /* Seen-time */ + addReplyBulkCString(c,"seen-time"); + addReplyLongLong(c,consumer->seen_time); + + /* Consumer PEL count */ + addReplyBulkCString(c,"pel-count"); + addReplyLongLong(c,raxSize(consumer->pel)); + + /* Consumer PEL */ + addReplyBulkCString(c,"pending"); + long long arraylen_cpel = 0; + void *arrayptr_cpel = addReplyDeferredLen(c); + raxIterator ri_cpel; + raxStart(&ri_cpel,consumer->pel); + raxSeek(&ri_cpel,"^",NULL,0); + while(raxNext(&ri_cpel) && (!count || arraylen_cpel < count)) { + streamNACK *nack = ri_cpel.data; + addReplyArrayLen(c,3); + + /* Entry ID. */ + streamID id; + streamDecodeID(ri_cpel.key,&id); + addReplyStreamID(c,&id); + + /* Last delivery. */ + addReplyLongLong(c,nack->delivery_time); + + /* Number of deliveries. */ + addReplyLongLong(c,nack->delivery_count); + + arraylen_cpel++; + } + setDeferredArrayLen(c,arrayptr_cpel,arraylen_cpel); + raxStop(&ri_cpel); + } + raxStop(&ri_consumers); + } + raxStop(&ri_cgroups); + } + } +} + /* XINFO CONSUMERS * XINFO GROUPS - * XINFO STREAM + * XINFO STREAM [FULL [COUNT ]] * XINFO HELP. */ void xinfoCommand(client *c) { const char *help[] = { -"CONSUMERS -- Show consumer groups of group .", -"GROUPS -- Show the stream consumer groups.", -"STREAM -- Show information about the stream.", -"HELP -- Print this help.", +"CONSUMERS -- Show consumer groups of group .", +"GROUPS -- Show the stream consumer groups.", +"STREAM [FULL [COUNT ]] -- Show information about the stream.", +"HELP -- Print this help.", NULL }; stream *s = NULL; @@ -2564,36 +2748,10 @@ NULL addReplyStreamID(c,&cg->last_id); } raxStop(&ri); - } else if (!strcasecmp(opt,"STREAM") && c->argc == 3) { - /* XINFO STREAM (or the alias XINFO ). */ - addReplyMapLen(c,7); - addReplyBulkCString(c,"length"); - addReplyLongLong(c,s->length); - addReplyBulkCString(c,"radix-tree-keys"); - addReplyLongLong(c,raxSize(s->rax)); - addReplyBulkCString(c,"radix-tree-nodes"); - addReplyLongLong(c,s->rax->numnodes); - addReplyBulkCString(c,"groups"); - addReplyLongLong(c,s->cgroups ? raxSize(s->cgroups) : 0); - addReplyBulkCString(c,"last-generated-id"); - addReplyStreamID(c,&s->last_id); - - /* To emit the first/last entry we us the streamReplyWithRange() - * API. */ - int count; - streamID start, end; - start.ms = start.seq = 0; - end.ms = end.seq = UINT64_MAX; - addReplyBulkCString(c,"first-entry"); - count = streamReplyWithRange(c,s,&start,&end,1,0,NULL,NULL, - STREAM_RWR_RAWENTRIES,NULL); - if (!count) addReplyNull(c); - addReplyBulkCString(c,"last-entry"); - count = streamReplyWithRange(c,s,&start,&end,1,1,NULL,NULL, - STREAM_RWR_RAWENTRIES,NULL); - if (!count) addReplyNull(c); + } else if (!strcasecmp(opt,"STREAM")) { + /* XINFO STREAM [FULL [COUNT ]]. */ + xinfoReplyWithStreamInfo(c,s); } else { addReplySubcommandSyntaxError(c); } } - diff --git a/tests/unit/type/stream-cgroups.tcl b/tests/unit/type/stream-cgroups.tcl index 04661707b..dfcd735f6 100644 --- a/tests/unit/type/stream-cgroups.tcl +++ b/tests/unit/type/stream-cgroups.tcl @@ -294,6 +294,40 @@ start_server { assert {[lindex $reply 0 3] == 2} } + test {XINFO FULL output} { + r del x + r XADD x 100 a 1 + r XADD x 101 b 1 + r XADD x 102 c 1 + r XADD x 103 e 1 + r XADD x 104 f 1 + r XGROUP CREATE x g1 0 + r XGROUP CREATE x g2 0 + r XREADGROUP GROUP g1 Alice COUNT 1 STREAMS x > + r XREADGROUP GROUP g1 Bob COUNT 1 STREAMS x > + r XREADGROUP GROUP g1 Bob NOACK COUNT 1 STREAMS x > + r XREADGROUP GROUP g2 Charlie COUNT 4 STREAMS x > + r XDEL x 103 + + set reply [r XINFO STREAM x FULL] + assert_equal [llength $reply] 12 + assert_equal [lindex $reply 1] 4 ;# stream length + assert_equal [lindex $reply 9] "{100-0 {a 1}} {101-0 {b 1}} {102-0 {c 1}} {104-0 {f 1}}" ;# entries + assert_equal [lindex $reply 11 0 1] "g1" ;# first group name + assert_equal [lindex $reply 11 0 7 0 0] "100-0" ;# first entry in group's PEL + assert_equal [lindex $reply 11 0 9 0 1] "Alice" ;# first consumer + assert_equal [lindex $reply 11 0 9 0 7 0 0] "100-0" ;# first entry in first consumer's PEL + assert_equal [lindex $reply 11 1 1] "g2" ;# second group name + assert_equal [lindex $reply 11 1 9 0 1] "Charlie" ;# first consumer + assert_equal [lindex $reply 11 1 9 0 7 0 0] "100-0" ;# first entry in first consumer's PEL + assert_equal [lindex $reply 11 1 9 0 7 1 0] "101-0" ;# second entry in first consumer's PEL + + set reply [r XINFO STREAM x FULL COUNT 1] + assert_equal [llength $reply] 12 + assert_equal [lindex $reply 1] 4 + assert_equal [lindex $reply 9] "{100-0 {a 1}}" + } + start_server {} { set master [srv -1 client] set master_host [srv -1 host] From 9ec60a50c69f843485b291ecea1ba4af9e020c69 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 28 Apr 2020 16:40:15 +0200 Subject: [PATCH 0319/1280] Fix create-cluster BIN_PATH. --- utils/create-cluster/create-cluster | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/create-cluster/create-cluster b/utils/create-cluster/create-cluster index 4db0a6619..931f6f521 100755 --- a/utils/create-cluster/create-cluster +++ b/utils/create-cluster/create-cluster @@ -1,7 +1,7 @@ #!/bin/bash # Settings -BIN_PATH="../../" +BIN_PATH="../../src/" CLUSTER_HOST=127.0.0.1 PORT=30000 TIMEOUT=2000 From 6544cf0f48dc7789e3974392529efd139249dc43 Mon Sep 17 00:00:00 2001 From: Guy Benoish Date: Tue, 28 Apr 2020 17:58:25 +0300 Subject: [PATCH 0320/1280] XINFO STREAM FULL should have a default COUNT of 10 --- src/t_stream.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/t_stream.c b/src/t_stream.c index d6a5e0009..5c1b9a523 100644 --- a/src/t_stream.c +++ b/src/t_stream.c @@ -2493,7 +2493,7 @@ void xtrimCommand(client *c) { * Handles the variants of XINFO STREAM */ void xinfoReplyWithStreamInfo(client *c, stream *s) { int full = 1; - long long count = 0; + long long count = 10; /* Default COUNT is 10 so we don't block the server */ robj **optv = c->argv + 3; /* Options start after XINFO STREAM */ int optc = c->argc - 3; @@ -2521,7 +2521,7 @@ void xinfoReplyWithStreamInfo(client *c, stream *s) { } if (getLongLongFromObjectOrReply(c,optv[2],&count,NULL) == C_ERR) return; - if (count < 0) count = 0; + if (count < 0) count = 10; } } @@ -2548,11 +2548,11 @@ void xinfoReplyWithStreamInfo(client *c, stream *s) { end.ms = end.seq = UINT64_MAX; addReplyBulkCString(c,"first-entry"); emitted = streamReplyWithRange(c,s,&start,&end,1,0,NULL,NULL, - STREAM_RWR_RAWENTRIES,NULL); + STREAM_RWR_RAWENTRIES,NULL); if (!emitted) addReplyNull(c); addReplyBulkCString(c,"last-entry"); emitted = streamReplyWithRange(c,s,&start,&end,1,1,NULL,NULL, - STREAM_RWR_RAWENTRIES,NULL); + STREAM_RWR_RAWENTRIES,NULL); if (!emitted) addReplyNull(c); } else { /* XINFO STREAM FULL [COUNT ] */ @@ -2682,6 +2682,10 @@ void xinfoCommand(client *c) { "CONSUMERS -- Show consumer groups of group .", "GROUPS -- Show the stream consumer groups.", "STREAM [FULL [COUNT ]] -- Show information about the stream.", +" FULL will return the full state of the stream,", +" including all entries, groups, consumers and PELs.", +" It's possible to show only the first stream/PEL entries", +" by using the COUNT modifier (Default is 10)", "HELP -- Print this help.", NULL }; From e9d4c24e01f7f3075a1af41518bdb22bc51d588a Mon Sep 17 00:00:00 2001 From: srzhao Date: Tue, 26 Nov 2019 10:43:57 +0800 Subject: [PATCH 0321/1280] fix pipelined WAIT performance issue. If client gets blocked again in `processUnblockedClients`, redis will not send `REPLCONF GETACK *` to slaves untill next eventloop, so the client will be blocked for 100ms by default(10hz) if no other file event fired. move server.get_ack_from_slaves sinppet after `processUnblockedClients`, so that both the first WAIT command that puts client in blocked context and the following WAIT command processed in processUnblockedClients would trigger redis-sever to send `REPLCONF GETACK *`, so that the eventloop would get `REPLCONG ACK ` from slaves and unblocked ASAP. --- src/server.c | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/src/server.c b/src/server.c index ab6280121..150476276 100644 --- a/src/server.c +++ b/src/server.c @@ -2110,6 +2110,19 @@ void beforeSleep(struct aeEventLoop *eventLoop) { if (server.active_expire_enabled && server.masterhost == NULL) activeExpireCycle(ACTIVE_EXPIRE_CYCLE_FAST); + /* Unblock all the clients blocked for synchronous replication + * in WAIT. */ + if (listLength(server.clients_waiting_acks)) + processClientsWaitingReplicas(); + + /* Check if there are clients unblocked by modules that implement + * blocking commands. */ + if (moduleCount()) moduleHandleBlockedClients(); + + /* Try to process pending commands for clients that were just unblocked. */ + if (listLength(server.unblocked_clients)) + processUnblockedClients(); + /* Send all the slaves an ACK request if at least one client blocked * during the previous event loop iteration. */ if (server.get_ack_from_slaves) { @@ -2125,19 +2138,6 @@ void beforeSleep(struct aeEventLoop *eventLoop) { server.get_ack_from_slaves = 0; } - /* Unblock all the clients blocked for synchronous replication - * in WAIT. */ - if (listLength(server.clients_waiting_acks)) - processClientsWaitingReplicas(); - - /* Check if there are clients unblocked by modules that implement - * blocking commands. */ - if (moduleCount()) moduleHandleBlockedClients(); - - /* Try to process pending commands for clients that were just unblocked. */ - if (listLength(server.unblocked_clients)) - processUnblockedClients(); - /* Send the invalidation messages to clients participating to the * client side caching protocol in broadcasting (BCAST) mode. */ trackingBroadcastInvalidationMessages(); From e978e7140240accf0d31bf267502c284982c29de Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 29 Apr 2020 11:16:30 +0200 Subject: [PATCH 0322/1280] Comment clearly why we moved some code in #6623. --- src/server.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/server.c b/src/server.c index 150476276..659604ef3 100644 --- a/src/server.c +++ b/src/server.c @@ -2124,7 +2124,10 @@ void beforeSleep(struct aeEventLoop *eventLoop) { processUnblockedClients(); /* Send all the slaves an ACK request if at least one client blocked - * during the previous event loop iteration. */ + * during the previous event loop iteration. Note that we do this after + * processUnblockedClients(), so if there are multiple pipelined WAITs + * and the just unblocked WAIT gets blocked again, we don't have to wait + * a server cron cycle in absence of other event loop events. See #6623. */ if (server.get_ack_from_slaves) { robj *argv[3]; From a9cc31e32e092268b11db421c3fd0297935d00a1 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 29 Apr 2020 12:37:12 +0200 Subject: [PATCH 0323/1280] redis-cli: try to make clusterManagerFixOpenSlot() more readable. Also improve the message to make clear that there is no *clear* owner, not that there is no owner at all. --- src/redis-cli.c | 31 +++++++++++++++++++++++++------ 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/src/redis-cli.c b/src/redis-cli.c index 72480d08c..469dbb0ff 100644 --- a/src/redis-cli.c +++ b/src/redis-cli.c @@ -4596,20 +4596,26 @@ static int clusterManagerFixOpenSlot(int slot) { /* Try to obtain the current slot owner, according to the current * nodes configuration. */ int success = 1; - list *owners = listCreate(); + list *owners = listCreate(); /* List of nodes claiming some ownership. + it could be stating in the configuration + to have the node ownership, or just + holding keys for such slot. */ list *migrating = listCreate(); list *importing = listCreate(); sds migrating_str = sdsempty(); sds importing_str = sdsempty(); - clusterManagerNode *owner = NULL; + clusterManagerNode *owner = NULL; /* The obvious slot owner if any. */ + + /* Iterate all the nodes, looking for potential owners of this slot. */ listIter li; listNode *ln; listRewind(cluster_manager.nodes, &li); while ((ln = listNext(&li)) != NULL) { clusterManagerNode *n = ln->value; if (n->flags & CLUSTER_MANAGER_FLAG_SLAVE) continue; - if (n->slots[slot]) listAddNodeTail(owners, n); - else { + if (n->slots[slot]) { + listAddNodeTail(owners, n); + } else { redisReply *r = CLUSTER_MANAGER_COMMAND(n, "CLUSTER COUNTKEYSINSLOT %d", slot); success = clusterManagerCheckRedisReply(n, r, NULL); @@ -4623,7 +4629,14 @@ static int clusterManagerFixOpenSlot(int slot) { if (!success) goto cleanup; } } + + /* If we have only a single potential owner for this slot, + * set it as "owner". */ if (listLength(owners) == 1) owner = listFirst(owners)->value; + + /* Scan the list of nodes again, in order to populate the + * list of nodes in importing or migrating state for + * this slot. */ listRewind(cluster_manager.nodes, &li); while ((ln = listNext(&li)) != NULL) { clusterManagerNode *n = ln->value; @@ -4655,6 +4668,7 @@ static int clusterManagerFixOpenSlot(int slot) { } } } + /* If the node is neither migrating nor importing and it's not * the owner, then is added to the importing list in case * it has keys in the slot. */ @@ -4679,11 +4693,12 @@ static int clusterManagerFixOpenSlot(int slot) { printf("Set as migrating in: %s\n", migrating_str); if (sdslen(importing_str) > 0) printf("Set as importing in: %s\n", importing_str); + /* If there is no slot owner, set as owner the node with the biggest * number of keys, among the set of migrating / importing nodes. */ if (owner == NULL) { - clusterManagerLogInfo(">>> Nobody claims ownership, " - "selecting an owner...\n"); + clusterManagerLogInfo(">>> No single clear owner for the slot, " + "selecting an owner by # of keys...\n"); owner = clusterManagerGetNodeWithMostKeysInSlot(cluster_manager.nodes, slot, NULL); // If we still don't have an owner, we can't fix it. @@ -4714,6 +4729,7 @@ static int clusterManagerFixOpenSlot(int slot) { clusterManagerRemoveNodeFromList(migrating, owner); clusterManagerRemoveNodeFromList(importing, owner); } + /* If there are multiple owners of the slot, we need to fix it * so that a single node is the owner and all the other nodes * are in importing state. Later the fix can be handled by one @@ -4746,6 +4762,7 @@ static int clusterManagerFixOpenSlot(int slot) { } } int move_opts = CLUSTER_MANAGER_OPT_VERBOSE; + /* Case 1: The slot is in migrating state in one node, and in * importing state in 1 node. That's trivial to address. */ if (listLength(migrating) == 1 && listLength(importing) == 1) { @@ -4757,6 +4774,7 @@ static int clusterManagerFixOpenSlot(int slot) { move_opts |= CLUSTER_MANAGER_OPT_UPDATE; success = clusterManagerMoveSlot(src, dst, slot, move_opts, NULL); } + /* Case 2: There are multiple nodes that claim the slot as importing, * they probably got keys about the slot after a restart so opened * the slot. In this case we just move all the keys to the owner @@ -4787,6 +4805,7 @@ static int clusterManagerFixOpenSlot(int slot) { if (!success) goto cleanup; } } + /* Case 3: The slot is in migrating state in one node but multiple * other nodes claim to be in importing state and don't have any key in * the slot. We search for the importing node having the same ID as From ad291c381c464631f5a66c65db7e96d7ec142253 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 29 Apr 2020 16:28:16 +0200 Subject: [PATCH 0324/1280] redis-cli: simplify cluster nodes coverage display. --- src/redis-cli.c | 27 ++++++++++----------------- 1 file changed, 10 insertions(+), 17 deletions(-) diff --git a/src/redis-cli.c b/src/redis-cli.c index 469dbb0ff..82a46216c 100644 --- a/src/redis-cli.c +++ b/src/redis-cli.c @@ -4295,17 +4295,18 @@ static int clusterManagerGetCoveredSlots(char *all_slots) { } static void clusterManagerPrintSlotsList(list *slots) { + clusterManagerNode n = {0}; listIter li; listNode *ln; listRewind(slots, &li); - sds first = NULL; while ((ln = listNext(&li)) != NULL) { - sds slot = ln->value; - if (!first) first = slot; - else printf(", "); - printf("%s", slot); + int slot = atoi(ln->value); + if (slot >= 0 && slot < CLUSTER_MANAGER_SLOTS) + n.slots[slot] = 1; } - printf("\n"); + sds nodeslist = clusterManagerNodeSlotsString(&n); + printf("%s\n", nodeslist); + sdsfree(nodeslist); } /* Return the node, among 'nodes' with the greatest number of keys @@ -4398,15 +4399,10 @@ static int clusterManagerFixSlotsCoverage(char *all_slots) { int i, fixed = 0; list *none = NULL, *single = NULL, *multi = NULL; clusterManagerLogInfo(">>> Fixing slots coverage...\n"); - printf("List of not covered slots: \n"); - int uncovered_count = 0; - sds log = sdsempty(); for (i = 0; i < CLUSTER_MANAGER_SLOTS; i++) { int covered = all_slots[i]; if (!covered) { - sds key = sdsfromlonglong((long long) i); - if (uncovered_count++ > 0) printf(","); - printf("%s", (char *) key); + sds slot = sdsfromlonglong((long long) i); list *slot_nodes = listCreate(); sds slot_nodes_str = sdsempty(); listIter li; @@ -4433,13 +4429,11 @@ static int clusterManagerFixSlotsCoverage(char *all_slots) { } freeReplyObject(reply); } - log = sdscatfmt(log, "\nSlot %S has keys in %u nodes: %S", - key, listLength(slot_nodes), slot_nodes_str); sdsfree(slot_nodes_str); - dictAdd(clusterManagerUncoveredSlots, key, slot_nodes); + dictAdd(clusterManagerUncoveredSlots, slot, slot_nodes); } } - printf("\n%s\n", log); + /* For every slot, take action depending on the actual condition: * 1) No node has keys for this slot. * 2) A single node has keys for this slot. @@ -4581,7 +4575,6 @@ static int clusterManagerFixSlotsCoverage(char *all_slots) { } } cleanup: - sdsfree(log); if (none) listRelease(none); if (single) listRelease(single); if (multi) listRelease(multi); From 0e450be7cf26433bfe4abb18b0b3703be9efaa1e Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 29 Apr 2020 16:57:06 +0200 Subject: [PATCH 0325/1280] redis-cli: safer cluster fix with unreachalbe masters. --- src/redis-cli.c | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/src/redis-cli.c b/src/redis-cli.c index 82a46216c..ec0f58f5f 100644 --- a/src/redis-cli.c +++ b/src/redis-cli.c @@ -123,6 +123,7 @@ #define CLUSTER_MANAGER_CMD_FLAG_COPY 1 << 7 #define CLUSTER_MANAGER_CMD_FLAG_COLOR 1 << 8 #define CLUSTER_MANAGER_CMD_FLAG_CHECK_OWNERS 1 << 9 +#define CLUSTER_MANAGER_CMD_FLAG_FIX_WITH_UNREACHABLE_MASTERS 1 << 10 #define CLUSTER_MANAGER_OPT_GETFRIENDS 1 << 0 #define CLUSTER_MANAGER_OPT_COLD 1 << 1 @@ -1599,6 +1600,9 @@ static int parseOptions(int argc, char **argv) { } else if (!strcmp(argv[i],"--cluster-search-multiple-owners")) { config.cluster_manager_command.flags |= CLUSTER_MANAGER_CMD_FLAG_CHECK_OWNERS; + } else if (!strcmp(argv[i],"--cluster-fix-with-unreachable-masters")) { + config.cluster_manager_command.flags |= + CLUSTER_MANAGER_CMD_FLAG_FIX_WITH_UNREACHABLE_MASTERS; #ifdef USE_OPENSSL } else if (!strcmp(argv[i],"--tls")) { config.tls = 1; @@ -2146,6 +2150,7 @@ static int evalMode(int argc, char **argv) { static struct clusterManager { list *nodes; /* List of nodes in the configuration. */ list *errors; + int unreachable_masters; /* Masters we are not able to reach. */ } cluster_manager; /* Used by clusterManagerFixSlotsCoverage */ @@ -2288,7 +2293,7 @@ clusterManagerCommandDef clusterManagerCommands[] = { "search-multiple-owners"}, {"info", clusterManagerCommandInfo, -1, "host:port", NULL}, {"fix", clusterManagerCommandFix, -1, "host:port", - "search-multiple-owners"}, + "search-multiple-owners,fix-with-unreachable-masters"}, {"reshard", clusterManagerCommandReshard, -1, "host:port", "from ,to ,slots ,yes,timeout ,pipeline ," "replace"}, @@ -4013,7 +4018,9 @@ static int clusterManagerLoadInfoFromNode(clusterManagerNode *node, int opts) { if (friend->flags & (CLUSTER_MANAGER_FLAG_NOADDR | CLUSTER_MANAGER_FLAG_DISCONNECT | CLUSTER_MANAGER_FLAG_FAIL)) + { goto invalid_friend; + } listAddNodeTail(cluster_manager.nodes, friend); } else { clusterManagerLogErr("[ERR] Unable to load info for " @@ -4023,6 +4030,8 @@ static int clusterManagerLoadInfoFromNode(clusterManagerNode *node, int opts) { } continue; invalid_friend: + if (!(friend->flags & CLUSTER_MANAGER_FLAG_SLAVE)) + cluster_manager.unreachable_masters++; freeClusterManagerNode(friend); } listRelease(node->friends); @@ -4396,6 +4405,14 @@ static clusterManagerNode *clusterManagerNodeMasterRandom() { } static int clusterManagerFixSlotsCoverage(char *all_slots) { + int force_fix = config.cluster_manager_command.flags & + CLUSTER_MANAGER_CMD_FLAG_FIX_WITH_UNREACHABLE_MASTERS; + + if (cluster_manager.unreachable_masters > 0 && !force_fix) { + clusterManagerLogWarn("*** Fixing slots coverage with %d unreachable masters is dangerous: redis-cli will assume that slots about masters that are not reachable are not covered, and will try to reassign them to the reachable nodes. This can cause data loss and is rarely what you want to do. If you really want to proceed use the --cluster-fix-with-unreachable-masters option.\n", cluster_manager.unreachable_masters); + exit(1); + } + int i, fixed = 0; list *none = NULL, *single = NULL, *multi = NULL; clusterManagerLogInfo(">>> Fixing slots coverage...\n"); @@ -4585,6 +4602,14 @@ cleanup: * more nodes. This function fixes this condition by migrating keys where * it seems more sensible. */ static int clusterManagerFixOpenSlot(int slot) { + int force_fix = config.cluster_manager_command.flags & + CLUSTER_MANAGER_CMD_FLAG_FIX_WITH_UNREACHABLE_MASTERS; + + if (cluster_manager.unreachable_masters > 0 && !force_fix) { + clusterManagerLogWarn("*** Fixing open slots with %d unreachable masters is dangerous: redis-cli will assume that slots about masters that are not reachable are not covered, and will try to reassign them to the reachable nodes. This can cause data loss and is rarely what you want to do. If you really want to proceed use the --cluster-fix-with-unreachable-masters option.\n", cluster_manager.unreachable_masters); + exit(1); + } + clusterManagerLogInfo(">>> Fixing open slot %d\n", slot); /* Try to obtain the current slot owner, according to the current * nodes configuration. */ From 077dfaad724b38813db77a736ff4a440d4b7b68f Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 29 Apr 2020 18:49:42 +0200 Subject: [PATCH 0326/1280] Fix tracking table max keys option in redis.conf. --- redis.conf | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/redis.conf b/redis.conf index 01a2d801f..d96d26e1c 100644 --- a/redis.conf +++ b/redis.conf @@ -626,20 +626,23 @@ replica-priority 100 # to track the keys fetched by many clients. # # For this reason it is possible to configure a maximum fill value for the -# invalidation table. By default it is set to 10%, and once this limit is -# reached, Redis will start to evict caching slots in the invalidation table -# even if keys are not modified, just to reclaim memory: this will in turn +# invalidation table. By default it is set to 1M of keys, and once this limit +# is reached, Redis will start to evict keys in the invalidation table +# even if they were not modified, just to reclaim memory: this will in turn # force the clients to invalidate the cached values. Basically the table -# maximum fill rate is a trade off between the memory you want to spend server +# maximum size is a trade off between the memory you want to spend server # side to track information about who cached what, and the ability of clients # to retain cached objects in memory. # -# If you set the value to 0, it means there are no limits, and all the 16 -# millions of caching slots can be used at the same time. In the "stats" -# INFO section, you can find information about the amount of caching slots -# used at every given moment. +# If you set the value to 0, it means there are no limits, and Redis will +# retain as many keys as needed in the invalidation table. +# In the "stats" INFO section, you can find information about the number of +# keys in the invalidation table at every given moment. # -# tracking-table-max-fill 10 +# Note: when key tracking is used in broadcasting mode, no memory is used +# in the server side so this setting is useless. +# +# tracking-table-max-keys 1000000 ################################## SECURITY ################################### From f55fc50dde2cb0c39ac9f6a9f82153b331c1af83 Mon Sep 17 00:00:00 2001 From: "zhaozhao.zz" Date: Wed, 27 Feb 2019 19:38:03 +0800 Subject: [PATCH 0327/1280] lazyfree & eviction: record latency generated by lazyfree eviction 1. add eviction-lazyfree monitor 2. put eviction-del & eviction-lazyfree into eviction-cycle that means eviction-cycle contains all the latency in the eviction cycle including del and lazyfree 3. use getMaxmemoryState to check if we can break in lazyfree-evict --- src/evict.c | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/src/evict.c b/src/evict.c index a829f1f99..0755acc0e 100644 --- a/src/evict.c +++ b/src/evict.c @@ -450,9 +450,10 @@ int freeMemoryIfNeeded(void) { if (server.masterhost && server.repl_slave_ignore_maxmemory) return C_OK; size_t mem_reported, mem_tofree, mem_freed; - mstime_t latency, eviction_latency; + mstime_t latency, eviction_latency, lazyfree_latency; long long delta; int slaves = listLength(server.slaves); + int result = C_ERR; /* When clients are paused the dataset should be static not just from the * POV of clients not being able to write, but also from the POV of @@ -463,10 +464,10 @@ int freeMemoryIfNeeded(void) { mem_freed = 0; + latencyStartMonitor(latency); if (server.maxmemory_policy == MAXMEMORY_NO_EVICTION) goto cant_free; /* We need to free memory, but policy forbids. */ - latencyStartMonitor(latency); while (mem_freed < mem_tofree) { int j, k, i; static unsigned int next_db = 0; @@ -572,7 +573,6 @@ int freeMemoryIfNeeded(void) { signalModifiedKey(NULL,db,keyobj); latencyEndMonitor(eviction_latency); latencyAddSampleIfNeeded("eviction-del",eviction_latency); - latencyRemoveNestedEvent(latency,eviction_latency); delta -= (long long) zmalloc_used_memory(); mem_freed += delta; server.stat_evictedkeys++; @@ -601,25 +601,30 @@ int freeMemoryIfNeeded(void) { } } } else { - latencyEndMonitor(latency); - latencyAddSampleIfNeeded("eviction-cycle",latency); goto cant_free; /* nothing to free... */ } } - latencyEndMonitor(latency); - latencyAddSampleIfNeeded("eviction-cycle",latency); - return C_OK; + result = C_OK; cant_free: /* We are here if we are not able to reclaim memory. There is only one * last thing we can try: check if the lazyfree thread has jobs in queue * and wait... */ - while(bioPendingJobsOfType(BIO_LAZY_FREE)) { - if (((mem_reported - zmalloc_used_memory()) + mem_freed) >= mem_tofree) - break; - usleep(1000); + if (result != C_OK) { + latencyStartMonitor(lazyfree_latency); + while(bioPendingJobsOfType(BIO_LAZY_FREE)) { + if (getMaxmemoryState(NULL,NULL,NULL,NULL) == C_OK) { + result = C_OK; + break; + } + usleep(1000); + } + latencyEndMonitor(lazyfree_latency); + latencyAddSampleIfNeeded("eviction-lazyfree",lazyfree_latency); } - return C_ERR; + latencyEndMonitor(latency); + latencyAddSampleIfNeeded("eviction-cycle",latency); + return result; } /* This is a wrapper for freeMemoryIfNeeded() that only really calls the From ccb24b9e16f87f0d8bf929c9c93170344ee89ea4 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 30 Apr 2020 09:58:06 +0200 Subject: [PATCH 0328/1280] CLIENT KILL USER . --- src/networking.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/networking.c b/src/networking.c index 7ffa99eb1..767206ab9 100644 --- a/src/networking.c +++ b/src/networking.c @@ -2101,6 +2101,7 @@ void clientCommand(client *c) { "KILL