From e3d4b30d09dc8dc1c560794750f28689cc4f510b Mon Sep 17 00:00:00 2001
From: Chen Tianjie <chentianjie.ctj@alibaba-inc.com>
Date: Thu, 31 Aug 2023 02:51:14 +0800
Subject: [PATCH] Add two stats to count client input and output buffer oom.
 (#12476)

Add these INFO metrics:
* client_query_buffer_limit_disconnections
* client_output_buffer_limit_disconnections

Sometimes it is useful to monitor whether clients reaches size limit of
query buffer and output buffer, to decide whether we need to adjust the
buffer size limit or reduce client query payload.
---
 src/networking.c    |  2 ++
 src/server.c        |  6 ++++++
 src/server.h        |  2 ++
 tests/unit/info.tcl | 21 +++++++++++++++++++++
 4 files changed, 31 insertions(+)

diff --git a/src/networking.c b/src/networking.c
index 7696e8c28..75f945334 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -2705,6 +2705,7 @@ void readQueryFromClient(connection *conn) {
         sdsfree(ci);
         sdsfree(bytes);
         freeClientAsync(c);
+        server.stat_client_qbuf_limit_disconnections++;
         goto done;
     }
 
@@ -3929,6 +3930,7 @@ int closeClientOnOutputBufferLimitReached(client *c, int async) {
                       client);
         }
         sdsfree(client);
+        server.stat_client_outbuf_limit_disconnections++;
         return  1;
     }
     return 0;
diff --git a/src/server.c b/src/server.c
index 72dad9bca..929f9b7cd 100644
--- a/src/server.c
+++ b/src/server.c
@@ -2522,6 +2522,8 @@ void resetServerStats(void) {
     atomicSet(server.stat_total_reads_processed, 0);
     server.stat_io_writes_processed = 0;
     atomicSet(server.stat_total_writes_processed, 0);
+    server.stat_client_qbuf_limit_disconnections = 0;
+    server.stat_client_outbuf_limit_disconnections = 0;
     for (j = 0; j < STATS_METRIC_COUNT; j++) {
         server.inst_metric[j].idx = 0;
         server.inst_metric[j].last_sample_base = 0;
@@ -5929,6 +5931,8 @@ sds genRedisInfoString(dict *section_dict, int all_sections, int everything) {
             "total_writes_processed:%lld\r\n"
             "io_threaded_reads_processed:%lld\r\n"
             "io_threaded_writes_processed:%lld\r\n"
+            "client_query_buffer_limit_disconnections:%lld\r\n"
+            "client_output_buffer_limit_disconnections:%lld\r\n"
             "reply_buffer_shrinks:%lld\r\n"
             "reply_buffer_expands:%lld\r\n"
             "eventloop_cycles:%llu\r\n"
@@ -5984,6 +5988,8 @@ sds genRedisInfoString(dict *section_dict, int all_sections, int everything) {
             stat_total_writes_processed,
             server.stat_io_reads_processed,
             server.stat_io_writes_processed,
+            server.stat_client_qbuf_limit_disconnections,
+            server.stat_client_outbuf_limit_disconnections,
             server.stat_reply_buffer_shrinks,
             server.stat_reply_buffer_expands,
             server.duration_stats[EL_DURATION_TYPE_EL].cnt,
diff --git a/src/server.h b/src/server.h
index 4f3005ae9..756435664 100644
--- a/src/server.h
+++ b/src/server.h
@@ -1695,6 +1695,8 @@ struct redisServer {
     long long stat_io_writes_processed; /* Number of write events processed by IO / Main threads */
     redisAtomic long long stat_total_reads_processed; /* Total number of read events processed */
     redisAtomic long long stat_total_writes_processed; /* Total number of write events processed */
+    long long stat_client_qbuf_limit_disconnections;  /* Total number of clients reached query buf length limit */
+    long long stat_client_outbuf_limit_disconnections;  /* Total number of clients reached output buf length limit */
     /* The following two are used to track instantaneous metrics, like
      * number of operations per second, network traffic. */
     struct {
diff --git a/tests/unit/info.tcl b/tests/unit/info.tcl
index 812704340..ae3f82699 100644
--- a/tests/unit/info.tcl
+++ b/tests/unit/info.tcl
@@ -342,5 +342,26 @@ start_server {tags {"info" "external:skip"}} {
             assert {$duration_max2 >= $duration_max1}
         }
 
+        test {stats: client input and output buffer limit disconnections} {
+            r config resetstat
+            set info [r info stats]
+            assert_equal [getInfoProperty $info client_query_buffer_limit_disconnections] {0}
+            assert_equal [getInfoProperty $info client_output_buffer_limit_disconnections] {0}
+            # set qbuf limit to minimum to test stat
+            set org_qbuf_limit [lindex [r config get client-query-buffer-limit] 1]
+            r config set client-query-buffer-limit 1048576
+            catch {r set key [string repeat a 1048576]}
+            set info [r info stats]
+            assert_equal [getInfoProperty $info client_query_buffer_limit_disconnections] {1}
+            r config set client-query-buffer-limit $org_qbuf_limit
+            # set outbuf limit to just 10 to test stat
+            set org_outbuf_limit [lindex [r config get client-output-buffer-limit] 1]
+            r config set client-output-buffer-limit "normal 10 0 0"
+            r set key [string repeat a 100000] ;# to trigger output buffer limit check this needs to be big
+            catch {r get key}
+            set info [r info stats]
+            assert_equal [getInfoProperty $info client_output_buffer_limit_disconnections] {1}
+            r config set client-output-buffer-limit $org_outbuf_limit
+        }
     }
 }