From bdff0121eb38c800cf79f43e78e15998821d0f05 Mon Sep 17 00:00:00 2001
From: John Sully <john@csquare.ca>
Date: Fri, 29 Oct 2021 17:59:46 +0000
Subject: [PATCH] Make the replica weighting configurable

Former-commit-id: be6a8a7e68acb5cfbe950f13b903e6f7b98c5a39
---
 keydb.conf                   | 11 +++++++++++
 src/config.cpp               |  1 +
 src/networking.cpp           |  3 ++-
 src/server.h                 |  2 ++
 tests/unit/introspection.tcl |  5 +++++
 5 files changed, 21 insertions(+), 1 deletion(-)

diff --git a/keydb.conf b/keydb.conf
index 50f9784a1..352f07663 100644
--- a/keydb.conf
+++ b/keydb.conf
@@ -1834,3 +1834,14 @@ server-threads 2
 
 # Enable FLASH support? (Enterprise Only)
 # storage-provider flash /path/to/flash/db
+
+# KeyDB will attempt to balance clients across threads evenly; However, replica clients
+# are usually much more expensive than a normal client, and so KeyDB will try to assign
+# fewer clients to threads with a replica.  The weighting factor below is intented to help tune
+# this behavior.  A replica weighting factor of 2 means we treat a replica as the equivalent
+# of two normal clients.  Adjusting this value may improve performance when replication is
+# used.  The best weighting is workload specific - e.g. read heavy workloads should set
+# this to 1.  Very write heavy workloads may benefit from higher numbers.
+#
+# By default KeyDB sets this to 2.
+replica-weighting-factor 2
\ No newline at end of file
diff --git a/src/config.cpp b/src/config.cpp
index 701565e1e..e2028f4cc 100644
--- a/src/config.cpp
+++ b/src/config.cpp
@@ -2803,6 +2803,7 @@ standardConfig configs[] = {
     createIntConfig("min-clients-per-thread", NULL, MODIFIABLE_CONFIG, 0, 400, cserver.thread_min_client_threshold, 20, INTEGER_CONFIG, NULL, NULL),
     createIntConfig("storage-flush-period", NULL, MODIFIABLE_CONFIG, 1, 10000, g_pserver->storage_flush_period, 500, INTEGER_CONFIG, NULL, NULL),
     createIntConfig("replica-quorum", NULL, MODIFIABLE_CONFIG, -1, INT_MAX, g_pserver->repl_quorum, -1, INTEGER_CONFIG, NULL, NULL),
+    createIntConfig("replica-weighting-factor", NULL, MODIFIABLE_CONFIG, 1, INT_MAX, g_pserver->replicaIsolationFactor, 2, INTEGER_CONFIG, NULL, NULL),
     /* Unsigned int configs */
     createUIntConfig("maxclients", NULL, MODIFIABLE_CONFIG, 1, UINT_MAX, g_pserver->maxclients, 10000, INTEGER_CONFIG, NULL, updateMaxclients),
     createUIntConfig("loading-process-events-interval-keys", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, g_pserver->loading_process_events_interval_keys, 8192, MEMORY_CONFIG, NULL, NULL),
diff --git a/src/networking.cpp b/src/networking.cpp
index d1ccf28a9..29c4ac206 100644
--- a/src/networking.cpp
+++ b/src/networking.cpp
@@ -1144,7 +1144,8 @@ int chooseBestThreadForAccept()
         int cclientsThread;
         atomicGet(g_pserver->rgthreadvar[iel].cclients, cclientsThread);
         cclientsThread += rgacceptsInFlight[iel].load(std::memory_order_relaxed);
-        cclientsThread *= (g_pserver->rgthreadvar[iel].cclientsReplica+1);
+        // Note: Its repl factor less one because cclients also includes replicas, so we don't want to double count
+        cclientsThread += (g_pserver->rgthreadvar[iel].cclientsReplica) * (g_pserver->replicaIsolationFactor-1);
         if (cclientsThread < cserver.thread_min_client_threshold)
             return iel;
         if (cclientsThread < cclientsMin)
diff --git a/src/server.h b/src/server.h
index 1b2c02f04..8997c3e73 100644
--- a/src/server.h
+++ b/src/server.h
@@ -2216,6 +2216,8 @@ struct redisServer {
 
     int active_expire_enabled;      /* Can be disabled for testing purposes. */
 
+    int replicaIsolationFactor = 1;
+
     /* Fields used only for stats */
     long long stat_numcommands;     /* Number of processed commands */
     long long stat_numconnections;  /* Number of connections received */
diff --git a/tests/unit/introspection.tcl b/tests/unit/introspection.tcl
index 2a0474ed8..b562530f6 100644
--- a/tests/unit/introspection.tcl
+++ b/tests/unit/introspection.tcl
@@ -44,6 +44,11 @@ start_server {tags {"introspection"}} {
         set e
     } {ERR*}
 
+    test {replica-weighting-factor does not accept values less than 1} {
+        catch {r config set replica-weighting-factor 0} e
+        set e
+    } {ERR*}
+
     test {CLIENT SETNAME can assign a name to this connection} {
         assert_equal [r client setname myname] {OK}
         r client list