From 91f4f41665c4e9e0ad248ce0b528644de28d0acd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Loyet?= <822436+fatpat@users.noreply.github.com> Date: Tue, 30 Mar 2021 22:40:22 +0200 Subject: [PATCH] Add replica-announced config option (#8653) The 'sentinel replicas ' command will ignore replicas with `replica-announced` set to no. The goal of disabling the config setting replica-announced is to allow ghost replicas. The replica is in the cluster, synchronize with its master, can be promoted to master and is not exposed to sentinel clients. This way, it is acting as a live backup or living ghost. In addition, to prevent the replica to be promoted as master, set replica-priority to 0. --- redis.conf | 12 ++++ src/config.c | 1 + src/sentinel.c | 17 ++++- src/server.c | 6 +- src/server.h | 1 + tests/sentinel/tests/10-replica-priority.tcl | 73 ++++++++++++++++++++ 6 files changed, 107 insertions(+), 3 deletions(-) create mode 100644 tests/sentinel/tests/10-replica-priority.tcl diff --git a/redis.conf b/redis.conf index a78d958c3..2663cfce6 100644 --- a/redis.conf +++ b/redis.conf @@ -667,6 +667,18 @@ repl-disable-tcp-nodelay no # By default the priority is 100. replica-priority 100 +# ----------------------------------------------------------------------------- +# By default, Redis Sentinel includes all replicas in its reports. A replica +# can be excluded from Redis Sentinel's announcements. An unannounced replica +# will be ignored by the 'sentinel replicas ' command and won't be +# exposed to Redis Sentinel's clients. +# +# This option does not change the behavior of replica-priority. Even with +# replica-announced set to 'no', the replica can be promoted to master. To +# prevent this behavior, set replica-priority to 0. +# +# replica-announced yes + # It is possible for a master to stop accepting writes if there are less than # N replicas connected, having a lag less or equal than M seconds. # diff --git a/src/config.c b/src/config.c index 64058791c..a29f10e1e 100644 --- a/src/config.c +++ b/src/config.c @@ -2438,6 +2438,7 @@ standardConfig configs[] = { createBoolConfig("crash-memcheck-enabled", NULL, MODIFIABLE_CONFIG, server.memcheck_enabled, 1, NULL, NULL), createBoolConfig("use-exit-on-panic", NULL, MODIFIABLE_CONFIG, server.use_exit_on_panic, 0, NULL, NULL), createBoolConfig("disable-thp", NULL, MODIFIABLE_CONFIG, server.disable_thp, 1, NULL, NULL), + createBoolConfig("replica-announced", NULL, MODIFIABLE_CONFIG, server.replica_announced, 1, NULL, NULL), /* String Configs */ createStringConfig("aclfile", NULL, IMMUTABLE_CONFIG, ALLOW_EMPTY_STRING, server.acl_filename, "", NULL, NULL), diff --git a/src/sentinel.c b/src/sentinel.c index a23b5b328..892fbc9c9 100644 --- a/src/sentinel.c +++ b/src/sentinel.c @@ -215,6 +215,7 @@ typedef struct sentinelRedisInstance { /* Slave specific. */ mstime_t master_link_down_time; /* Slave replication link down time. */ int slave_priority; /* Slave priority according to its INFO output. */ + int replica_announced; /* Replica announcing according to its INFO output. */ mstime_t slave_reconf_sent_time; /* Time at which we sent SLAVE OF */ struct sentinelRedisInstance *master; /* Master instance if it's slave. */ char *slave_master_host; /* Master host as reported by INFO */ @@ -1335,6 +1336,7 @@ sentinelRedisInstance *createSentinelRedisInstance(char *name, int flags, char * ri->auth_pass = NULL; ri->auth_user = NULL; ri->slave_priority = SENTINEL_DEFAULT_SLAVE_PRIORITY; + ri->replica_announced = 1; ri->slave_reconf_sent_time = 0; ri->slave_master_host = NULL; ri->slave_master_port = 0; @@ -2622,6 +2624,10 @@ void sentinelRefreshInstanceInfo(sentinelRedisInstance *ri, const char *info) { /* slave_repl_offset: */ if (sdslen(l) >= 18 && !memcmp(l,"slave_repl_offset:",18)) ri->slave_repl_offset = strtoull(l+18,NULL,10); + + /* replica_announced: */ + if (sdslen(l) >= 18 && !memcmp(l,"replica_announced:",18)) + ri->replica_announced = atoi(l+18); } } ri->info_refresh = mstime(); @@ -3424,6 +3430,10 @@ void addReplySentinelRedisInstance(client *c, sentinelRedisInstance *ri) { addReplyBulkCString(c,"slave-repl-offset"); addReplyBulkLongLong(c,ri->slave_repl_offset); fields++; + + addReplyBulkCString(c,"replica-announced"); + addReplyBulkLongLong(c,ri->replica_announced); + fields++; } /* Only sentinels */ @@ -3449,15 +3459,20 @@ void addReplySentinelRedisInstance(client *c, sentinelRedisInstance *ri) { void addReplyDictOfRedisInstances(client *c, dict *instances) { dictIterator *di; dictEntry *de; + long slaves = 0; + void *replylen = addReplyDeferredLen(c); di = dictGetIterator(instances); - addReplyArrayLen(c,dictSize(instances)); while((de = dictNext(di)) != NULL) { sentinelRedisInstance *ri = dictGetVal(de); + /* don't announce unannounced replicas */ + if (ri->flags & SRI_SLAVE && !ri->replica_announced) continue; addReplySentinelRedisInstance(c,ri); + slaves++; } dictReleaseIterator(di); + setDeferredArrayLen(c, replylen, slaves); } /* Lookup the named master into sentinel.masters. diff --git a/src/server.c b/src/server.c index 3421b9303..25cb1b40a 100644 --- a/src/server.c +++ b/src/server.c @@ -5110,9 +5110,11 @@ sds genRedisInfoString(const char *section) { } info = sdscatprintf(info, "slave_priority:%d\r\n" - "slave_read_only:%d\r\n", + "slave_read_only:%d\r\n" + "replica_announced:%d\r\n", server.slave_priority, - server.repl_slave_ro); + server.repl_slave_ro, + server.replica_announced); } info = sdscatprintf(info, diff --git a/src/server.h b/src/server.h index 490c74f8d..407072972 100644 --- a/src/server.h +++ b/src/server.h @@ -1468,6 +1468,7 @@ struct redisServer { time_t repl_down_since; /* Unix time at which link with master went down */ int repl_disable_tcp_nodelay; /* Disable TCP_NODELAY after SYNC? */ int slave_priority; /* Reported in INFO and used by Sentinel. */ + int replica_announced; /* If true, replica is announced by Sentinel */ int slave_announce_port; /* Give the master this listening port. */ char *slave_announce_ip; /* Give the master this ip address. */ /* The following two fields is where we store master PSYNC replid/offset diff --git a/tests/sentinel/tests/10-replica-priority.tcl b/tests/sentinel/tests/10-replica-priority.tcl new file mode 100644 index 000000000..00248a734 --- /dev/null +++ b/tests/sentinel/tests/10-replica-priority.tcl @@ -0,0 +1,73 @@ +source "../tests/includes/init-tests.tcl" + +test "Check acceptable replica-priority values" { + foreach_redis_id id { + if {$id == $master_id} continue + + # ensure replica-announced accepts yes and no + catch {R $id CONFIG SET replica-announced no} e + if {$e ne "OK"} { + fail "Unable to set replica-announced to no" + } + catch {R $id CONFIG SET replica-announced yes} e + if {$e ne "OK"} { + fail "Unable to set replica-announced to yes" + } + + # ensure a random value throw error + catch {R $id CONFIG SET replica-announced 321} e + if {$e eq "OK"} { + fail "Able to set replica-announced with something else than yes or no (321) whereas it should not be possible" + } + catch {R $id CONFIG SET replica-announced a3b2c1} e + if {$e eq "OK"} { + fail "Able to set replica-announced with something else than yes or no (a3b2c1) whereas it should not be possible" + } + + # test only the first redis replica, no need to double test + break + } +} + +proc 10_test_number_of_replicas {n_replicas_expected} { + test "Check sentinel replies with $n_replicas_expected replicas" { + # ensure sentinels replies with the right number of replicas + foreach_sentinel_id id { + # retries 40 x 500ms = 20s as SENTINEL_INFO_PERIOD = 10s + set len [llength [S $id SENTINEL REPLICAS mymaster]] + wait_for_condition 40 500 { + [llength [S $id SENTINEL REPLICAS mymaster]] == $n_replicas_expected + } else { + fail "Sentinel replies with a wrong number of replicas with replica-announced=yes (expected $n_replicas_expected but got $len) on sentinel $id" + } + } + } +} + +proc 10_set_replica_announced {master_id announced n_replicas} { + test "Set replica-announced=$announced on $n_replicas replicas" { + set i 0 + foreach_redis_id id { + if {$id == $master_id} continue + #puts "set replica-announce=$announced on redis #$id" + R $id CONFIG SET replica-announced "$announced" + incr i + if { $n_replicas!="all" && $i >= $n_replicas } { break } + } + } +} + +# ensure all replicas are announced +10_set_replica_announced $master_id "yes" "all" +# ensure all replicas are announced by sentinels +10_test_number_of_replicas 4 + +# ensure the first 2 replicas are not announced +10_set_replica_announced $master_id "no" 2 +# ensure sentinels are not announcing the first 2 replicas that have been set unannounced +10_test_number_of_replicas 2 + +# ensure all replicas are announced +10_set_replica_announced $master_id "yes" "all" +# ensure all replicas are not announced by sentinels +10_test_number_of_replicas 4