From 4eecb1825f5d5be59682279b88bde9ade396a6e8 Mon Sep 17 00:00:00 2001
From: John Sully <john@csquare.ca>
Date: Wed, 16 Sep 2020 03:26:31 +0000
Subject: [PATCH] Implement replica-quorum config

Former-commit-id: ab4cdd2ed4d3ee9752737d20662178d73d16b4c2
---
 keydb.conf                                    |  7 +++++++
 src/config.cpp                                |  1 +
 src/replication.cpp                           | 12 +++++++++--
 src/server.h                                  |  1 +
 tests/integration/replication-multimaster.tcl | 20 +++++++++++++++++++
 5 files changed, 39 insertions(+), 2 deletions(-)

diff --git a/keydb.conf b/keydb.conf
index 53afa107a..4bdd96ac8 100644
--- a/keydb.conf
+++ b/keydb.conf
@@ -417,6 +417,13 @@ dir ./
 #
 replica-serve-stale-data yes
 
+# You can modify the number of masters necessary to form a replica quorum when
+# multi-master is enabled and replica-serve-stale-data is "no".  By default 
+# this is set to -1 which implies the number of known masters (e.g. those
+# you added with replicaof)
+#
+# replica-quorum -1
+
 # You can configure a replica instance to accept writes or not. Writing against
 # a replica instance may be useful to store some ephemeral data (because data
 # written on a replica will be easily deleted after resync with the master) but
diff --git a/src/config.cpp b/src/config.cpp
index c0b8ff87e..8392bfd52 100644
--- a/src/config.cpp
+++ b/src/config.cpp
@@ -2293,6 +2293,7 @@ standardConfig configs[] = {
     createIntConfig("min-replicas-to-write", "min-slaves-to-write", MODIFIABLE_CONFIG, 0, INT_MAX, g_pserver->repl_min_slaves_to_write, 0, INTEGER_CONFIG, NULL, updateGoodSlaves),
     createIntConfig("min-replicas-max-lag", "min-slaves-max-lag", MODIFIABLE_CONFIG, 0, INT_MAX, g_pserver->repl_min_slaves_max_lag, 10, INTEGER_CONFIG, NULL, updateGoodSlaves),
     createIntConfig("min-clients-per-thread", NULL, MODIFIABLE_CONFIG, 0, 400, cserver.thread_min_client_threshold, 50, INTEGER_CONFIG, NULL, NULL),
+    createIntConfig("replica-quorum", NULL, MODIFIABLE_CONFIG, -1, INT_MAX, g_pserver->repl_quorum, -1, INTEGER_CONFIG, NULL, NULL),
     /* Unsigned int configs */
     createUIntConfig("maxclients", NULL, MODIFIABLE_CONFIG, 1, UINT_MAX, g_pserver->maxclients, 10000, INTEGER_CONFIG, NULL, updateMaxclients),
 
diff --git a/src/replication.cpp b/src/replication.cpp
index 2299a7530..38bbed40f 100644
--- a/src/replication.cpp
+++ b/src/replication.cpp
@@ -4007,12 +4007,20 @@ int FBrokenLinkToMaster()
     listNode *ln;
     listRewind(g_pserver->masters, &li);
 
+    int connected = 0;
     while ((ln = listNext(&li)))
     {
         redisMaster *mi = (redisMaster*)listNodeValue(ln);
-        if (mi->repl_state != REPL_STATE_CONNECTED)
-            return true;
+        if (mi->repl_state == REPL_STATE_CONNECTED)
+            ++connected;
     }
+
+    if (g_pserver->repl_quorum < 0) {
+        return connected < (int)listLength(g_pserver->masters);
+    } else {
+        return connected < g_pserver->repl_quorum;
+    }
+
     return false;
 }
 
diff --git a/src/server.h b/src/server.h
index 8d2c7b929..7f228c478 100644
--- a/src/server.h
+++ b/src/server.h
@@ -1881,6 +1881,7 @@ struct redisServer {
     int repl_syncio_timeout; /* Timeout for synchronous I/O calls */
     int repl_disable_tcp_nodelay;   /* Disable TCP_NODELAY after SYNC? */
     int repl_serve_stale_data; /* Serve stale data when link is down? */
+    int repl_quorum;           /* For multimaster what do we consider a quorum? -1 means all master must be online */
     int repl_slave_ro;          /* Slave is read only? */
     int repl_slave_ignore_maxmemory;    /* If true slaves do not evict. */
     int slave_priority;             /* Reported in INFO and used by Sentinel. */
diff --git a/tests/integration/replication-multimaster.tcl b/tests/integration/replication-multimaster.tcl
index 0e753c147..0f0d34b7f 100644
--- a/tests/integration/replication-multimaster.tcl
+++ b/tests/integration/replication-multimaster.tcl
@@ -96,6 +96,26 @@ start_server {overrides {hz 500 active-replica yes multi-master yes}} {
             }
         }
     }
+
+    # Keep this test last since it muchs with the config
+    if [string equal $topology "mesh"] {
+    test "$topology_name quorum respected" {
+        $R(0) config set replica-serve-stale-data no
+
+        # No issues when all nodes are connected with default settings
+        $R(0) get testkey
+
+        # No issues when quorum is equal to the number of nodes
+        $R(0) config set replica-quorum 3
+        $R(0) get testkey
+
+        $R(0) config set replica-quorum 4
+        catch {
+            $R(0) get testkey
+        } e
+        assert_match {*MASTER is down*} $e
+    }
+    }
 }
 }
 }