Implement replica-quorum config

Former-commit-id: ab4cdd2ed4d3ee9752737d20662178d73d16b4c2

keydb.conf

@@ -417,6 +417,13 @@ dir ./
 #
 replica-serve-stale-data yes
 
+# You can modify the number of masters necessary to form a replica quorum when
+# multi-master is enabled and replica-serve-stale-data is "no".  By default 
+# this is set to -1 which implies the number of known masters (e.g. those
+# you added with replicaof)
+#
+# replica-quorum -1
+
 # You can configure a replica instance to accept writes or not. Writing against
 # a replica instance may be useful to store some ephemeral data (because data
 # written on a replica will be easily deleted after resync with the master) but

src/config.cpp

@@ -2293,6 +2293,7 @@ standardConfig configs[] = {
     createIntConfig("min-replicas-to-write", "min-slaves-to-write", MODIFIABLE_CONFIG, 0, INT_MAX, g_pserver->repl_min_slaves_to_write, 0, INTEGER_CONFIG, NULL, updateGoodSlaves),
     createIntConfig("min-replicas-max-lag", "min-slaves-max-lag", MODIFIABLE_CONFIG, 0, INT_MAX, g_pserver->repl_min_slaves_max_lag, 10, INTEGER_CONFIG, NULL, updateGoodSlaves),
     createIntConfig("min-clients-per-thread", NULL, MODIFIABLE_CONFIG, 0, 400, cserver.thread_min_client_threshold, 50, INTEGER_CONFIG, NULL, NULL),
+    createIntConfig("replica-quorum", NULL, MODIFIABLE_CONFIG, -1, INT_MAX, g_pserver->repl_quorum, -1, INTEGER_CONFIG, NULL, NULL),
     /* Unsigned int configs */
     createUIntConfig("maxclients", NULL, MODIFIABLE_CONFIG, 1, UINT_MAX, g_pserver->maxclients, 10000, INTEGER_CONFIG, NULL, updateMaxclients),
 

src/replication.cpp

@@ -4007,12 +4007,20 @@ int FBrokenLinkToMaster()
     listNode *ln;
     listRewind(g_pserver->masters, &li);
 
+    int connected = 0;
     while ((ln = listNext(&li)))
     {
         redisMaster *mi = (redisMaster*)listNodeValue(ln);
-        if (mi->repl_state != REPL_STATE_CONNECTED)
+        if (mi->repl_state == REPL_STATE_CONNECTED)
-            return true;
+            ++connected;
     }
+
+    if (g_pserver->repl_quorum < 0) {
+        return connected < (int)listLength(g_pserver->masters);
+    } else {
+        return connected < g_pserver->repl_quorum;
+    }
+
     return false;
 }
 

src/server.h

@@ -1881,6 +1881,7 @@ struct redisServer {
     int repl_syncio_timeout; /* Timeout for synchronous I/O calls */
     int repl_disable_tcp_nodelay;   /* Disable TCP_NODELAY after SYNC? */
     int repl_serve_stale_data; /* Serve stale data when link is down? */
+    int repl_quorum;           /* For multimaster what do we consider a quorum? -1 means all master must be online */
     int repl_slave_ro;          /* Slave is read only? */
     int repl_slave_ignore_maxmemory;    /* If true slaves do not evict. */
     int slave_priority;             /* Reported in INFO and used by Sentinel. */

tests/integration/replication-multimaster.tcl

@@ -96,6 +96,26 @@ start_server {overrides {hz 500 active-replica yes multi-master yes}} {
             }
         }
     }
+
+    # Keep this test last since it muchs with the config
+    if [string equal $topology "mesh"] {
+    test "$topology_name quorum respected" {
+        $R(0) config set replica-serve-stale-data no
+
+        # No issues when all nodes are connected with default settings
+        $R(0) get testkey
+
+        # No issues when quorum is equal to the number of nodes
+        $R(0) config set replica-quorum 3
+        $R(0) get testkey
+
+        $R(0) config set replica-quorum 4
+        catch {
+            $R(0) get testkey
+        } e
+        assert_match {*MASTER is down*} $e
+    }
+    }
 }
 }
 }