Add two stats to count client input and output buffer oom. (#12476)

Add these INFO metrics:
* client_query_buffer_limit_disconnections
* client_output_buffer_limit_disconnections

Sometimes it is useful to monitor whether clients reaches size limit of
query buffer and output buffer, to decide whether we need to adjust the
buffer size limit or reduce client query payload.
This commit is contained in:
Chen Tianjie 2023-08-31 02:51:14 +08:00 committed by GitHub
parent 4b281ce519
commit e3d4b30d09
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 31 additions and 0 deletions

View File

@ -2705,6 +2705,7 @@ void readQueryFromClient(connection *conn) {
sdsfree(ci);
sdsfree(bytes);
freeClientAsync(c);
server.stat_client_qbuf_limit_disconnections++;
goto done;
}
@ -3929,6 +3930,7 @@ int closeClientOnOutputBufferLimitReached(client *c, int async) {
client);
}
sdsfree(client);
server.stat_client_outbuf_limit_disconnections++;
return 1;
}
return 0;

View File

@ -2522,6 +2522,8 @@ void resetServerStats(void) {
atomicSet(server.stat_total_reads_processed, 0);
server.stat_io_writes_processed = 0;
atomicSet(server.stat_total_writes_processed, 0);
server.stat_client_qbuf_limit_disconnections = 0;
server.stat_client_outbuf_limit_disconnections = 0;
for (j = 0; j < STATS_METRIC_COUNT; j++) {
server.inst_metric[j].idx = 0;
server.inst_metric[j].last_sample_base = 0;
@ -5929,6 +5931,8 @@ sds genRedisInfoString(dict *section_dict, int all_sections, int everything) {
"total_writes_processed:%lld\r\n"
"io_threaded_reads_processed:%lld\r\n"
"io_threaded_writes_processed:%lld\r\n"
"client_query_buffer_limit_disconnections:%lld\r\n"
"client_output_buffer_limit_disconnections:%lld\r\n"
"reply_buffer_shrinks:%lld\r\n"
"reply_buffer_expands:%lld\r\n"
"eventloop_cycles:%llu\r\n"
@ -5984,6 +5988,8 @@ sds genRedisInfoString(dict *section_dict, int all_sections, int everything) {
stat_total_writes_processed,
server.stat_io_reads_processed,
server.stat_io_writes_processed,
server.stat_client_qbuf_limit_disconnections,
server.stat_client_outbuf_limit_disconnections,
server.stat_reply_buffer_shrinks,
server.stat_reply_buffer_expands,
server.duration_stats[EL_DURATION_TYPE_EL].cnt,

View File

@ -1695,6 +1695,8 @@ struct redisServer {
long long stat_io_writes_processed; /* Number of write events processed by IO / Main threads */
redisAtomic long long stat_total_reads_processed; /* Total number of read events processed */
redisAtomic long long stat_total_writes_processed; /* Total number of write events processed */
long long stat_client_qbuf_limit_disconnections; /* Total number of clients reached query buf length limit */
long long stat_client_outbuf_limit_disconnections; /* Total number of clients reached output buf length limit */
/* The following two are used to track instantaneous metrics, like
* number of operations per second, network traffic. */
struct {

View File

@ -342,5 +342,26 @@ start_server {tags {"info" "external:skip"}} {
assert {$duration_max2 >= $duration_max1}
}
test {stats: client input and output buffer limit disconnections} {
r config resetstat
set info [r info stats]
assert_equal [getInfoProperty $info client_query_buffer_limit_disconnections] {0}
assert_equal [getInfoProperty $info client_output_buffer_limit_disconnections] {0}
# set qbuf limit to minimum to test stat
set org_qbuf_limit [lindex [r config get client-query-buffer-limit] 1]
r config set client-query-buffer-limit 1048576
catch {r set key [string repeat a 1048576]}
set info [r info stats]
assert_equal [getInfoProperty $info client_query_buffer_limit_disconnections] {1}
r config set client-query-buffer-limit $org_qbuf_limit
# set outbuf limit to just 10 to test stat
set org_outbuf_limit [lindex [r config get client-output-buffer-limit] 1]
r config set client-output-buffer-limit "normal 10 0 0"
r set key [string repeat a 100000] ;# to trigger output buffer limit check this needs to be big
catch {r get key}
set info [r info stats]
assert_equal [getInfoProperty $info client_output_buffer_limit_disconnections] {1}
r config set client-output-buffer-limit $org_outbuf_limit
}
}
}