From c5f6cb1ba59f9c52036e3712447e62f4c906f10e Mon Sep 17 00:00:00 2001 From: John Sully Date: Sun, 12 Jul 2020 19:25:19 +0000 Subject: [PATCH] Add multi-master-no-forward command to reduce bus traffic with multi-master Former-commit-id: d99d06b1250a51ea4bc54f678f451acbb7901e33 --- keydb.conf | 6 ++++++ src/config.cpp | 8 ++++++++ src/replication.cpp | 2 +- src/server.h | 1 + tests/integration/replication-multimaster.tcl | 15 +++++++++++---- 5 files changed, 27 insertions(+), 5 deletions(-) diff --git a/keydb.conf b/keydb.conf index 43ce0c8a6..8979ee47c 100644 --- a/keydb.conf +++ b/keydb.conf @@ -1815,6 +1815,12 @@ jemalloc-bg-thread yes # Tuning this parameter is a tradeoff between locking overhead and distributing the workload over multiple cores # min-clients-per-thread 50 +# Avoid forwarding RREPLAY messages to other masters? +# WARNING: This setting is dangerous! You must be certain all masters are connected to each +# other in a true mesh topology or data loss will occur! +# This command can be used to reduce multimaster bus traffic +# multi-master-no-forward no + # Path to directory for file backed scratchpad. The file backed scratchpad # reduces memory requirements by storing rarely accessed data on disk # instead of RAM. A temporary file will be created in this directory. diff --git a/src/config.cpp b/src/config.cpp index 8b318420d..75360c4b7 100644 --- a/src/config.cpp +++ b/src/config.cpp @@ -2167,6 +2167,13 @@ static int updateMaxclients(long long val, long long prev, const char **err) { return 1; } +static int validateMultiMasterNoForward(int val, const char **) { + if (val) { + serverLog(LL_WARNING, "WARNING: multi-master-no-forward is set, you *must* use a mesh topology or dataloss will occur"); + } + return 1; +} + #ifdef USE_OPENSSL static int updateTlsCfg(char *val, char *prev, const char **err) { UNUSED(val); @@ -2222,6 +2229,7 @@ standardConfig configs[] = { createBoolConfig("cluster-enabled", NULL, IMMUTABLE_CONFIG, g_pserver->cluster_enabled, 0, NULL, NULL), createBoolConfig("appendonly", NULL, MODIFIABLE_CONFIG, g_pserver->aof_enabled, 0, NULL, updateAppendonly), createBoolConfig("cluster-allow-reads-when-down", NULL, MODIFIABLE_CONFIG, g_pserver->cluster_allow_reads_when_down, 0, NULL, NULL), + createBoolConfig("multi-master-no-forward", NULL, MODIFIABLE_CONFIG, cserver.multimaster_no_forward, 0, validateMultiMasterNoForward, NULL), /* String Configs */ createStringConfig("aclfile", NULL, IMMUTABLE_CONFIG, ALLOW_EMPTY_STRING, g_pserver->acl_filename, "", NULL, NULL), diff --git a/src/replication.cpp b/src/replication.cpp index b502cf0e7..c664b6012 100644 --- a/src/replication.cpp +++ b/src/replication.cpp @@ -4251,7 +4251,7 @@ void replicaReplayCommand(client *c) serverTL->current_client = current_clientSave; // call() will not propogate this for us, so we do so here - if (!s_pstate->FCancelled() && s_pstate->FFirst()) + if (!s_pstate->FCancelled() && s_pstate->FFirst() && !cserver.multimaster_no_forward) alsoPropagate(cserver.rreplayCommand,c->db->id,c->argv,c->argc,PROPAGATE_AOF|PROPAGATE_REPL); s_pstate->Pop(); diff --git a/src/server.h b/src/server.h index 3a6375b32..e6d73ab81 100644 --- a/src/server.h +++ b/src/server.h @@ -1648,6 +1648,7 @@ struct redisServerConst { unsigned char uuid[UUID_BINARY_LEN]; /* This server's UUID - populated on boot */ bool fUsePro = false; int thread_min_client_threshold = 50; + int multimaster_no_forward; }; struct redisServer { diff --git a/tests/integration/replication-multimaster.tcl b/tests/integration/replication-multimaster.tcl index 858cff26a..0e753c147 100644 --- a/tests/integration/replication-multimaster.tcl +++ b/tests/integration/replication-multimaster.tcl @@ -1,4 +1,6 @@ foreach topology {mesh ring} { + +foreach noforward [expr {[string equal $topology "mesh"] ? {no yes} : {no}}] { start_server {tags {"multi-master"} overrides {hz 500 active-replica yes multi-master yes}} { start_server {overrides {hz 500 active-replica yes multi-master yes}} { start_server {overrides {hz 500 active-replica yes multi-master yes}} { @@ -8,8 +10,12 @@ start_server {overrides {hz 500 active-replica yes multi-master yes}} { set R($j) [srv [expr 0-$j] client] set R_host($j) [srv [expr 0-$j] host] set R_port($j) [srv [expr 0-$j] port] + + $R($j) config set multi-master-no-forward $noforward } + set topology_name "$topology[expr {[string equal $noforward "yes"] ? " no-forward" : ""}]" + # Initialize as mesh if [string equal $topology "mesh"] { for {set j 0} {$j < 4} {incr j} { @@ -31,7 +37,7 @@ start_server {overrides {hz 500 active-replica yes multi-master yes}} { $R(3) replicaof $R_host(2) $R_port(2) } - test "$topology all nodes up" { + test "$topology_name all nodes up" { for {set j 0} {$j < 4} {incr j} { wait_for_condition 50 100 { [string match {*master_global_link_status:up*} [$R($j) info replication]] @@ -41,7 +47,7 @@ start_server {overrides {hz 500 active-replica yes multi-master yes}} { } } - test "$topology replicates to all nodes" { + test "$topology_name replicates to all nodes" { $R(0) set testkey foo after 500 for {set n 0} {$n < 4} {incr n} { @@ -53,7 +59,7 @@ start_server {overrides {hz 500 active-replica yes multi-master yes}} { } } - test "$topology replicates only once" { + test "$topology_name replicates only once" { $R(0) set testkey 1 after 500 #wait_for_condition 50 100 { @@ -74,7 +80,7 @@ start_server {overrides {hz 500 active-replica yes multi-master yes}} { } } - test "$topology transaction replicates only once" { + test "$topology_name transaction replicates only once" { for {set j 0} {$j < 1000} {incr j} { $R(0) set testkey 1 $R(0) multi @@ -95,3 +101,4 @@ start_server {overrides {hz 500 active-replica yes multi-master yes}} { } } } +}