From 17511df59b96bfeab8b46d474c19ec929e605bb9 Mon Sep 17 00:00:00 2001 From: Huang Zhw Date: Mon, 26 Jul 2021 15:07:20 +0800 Subject: [PATCH] Add INFO stat total_eviction_exceeded_time and current_eviction_exceeded_time (#9031) Add two INFO metrics: ``` total_eviction_exceeded_time:69734 current_eviction_exceeded_time:10230 ``` `current_eviction_exceeded_time` if greater than 0, means how much time current used memory is greater than `maxmemory`. And we are still over the maxmemory. If used memory is below `maxmemory`, this metric is reset to 0. `total_eviction_exceeded_time` means total time used memory is greater than `maxmemory` since server startup. The units of these two metrics are ms. Co-authored-by: Oran Agra --- src/evict.c | 26 +++++++++++++++++++++----- src/server.c | 8 ++++++++ src/server.h | 2 ++ 3 files changed, 31 insertions(+), 5 deletions(-) diff --git a/src/evict.c b/src/evict.c index 5fdcd4134..7a2befdd2 100644 --- a/src/evict.c +++ b/src/evict.c @@ -510,6 +510,8 @@ static unsigned long evictionTimeLimitUs() { * EVICT_FAIL - memory is over the limit, and there's nothing to evict * */ int performEvictions(void) { + /* Note, we don't goto update_metrics here because this check skips eviction + * as if it wasn't triggered. it's a fake EVICT_OK. */ if (!isSafeToPerformEvictions()) return EVICT_OK; int keys_freed = 0; @@ -520,11 +522,15 @@ int performEvictions(void) { int slaves = listLength(server.slaves); int result = EVICT_FAIL; - if (getMaxmemoryState(&mem_reported,NULL,&mem_tofree,NULL) == C_OK) - return EVICT_OK; + if (getMaxmemoryState(&mem_reported,NULL,&mem_tofree,NULL) == C_OK) { + result = EVICT_OK; + goto update_metrics; + } - if (server.maxmemory_policy == MAXMEMORY_NO_EVICTION) - return EVICT_FAIL; /* We need to free memory, but policy forbids. */ + if (server.maxmemory_policy == MAXMEMORY_NO_EVICTION) { + result = EVICT_FAIL; /* We need to free memory, but policy forbids. */ + goto update_metrics; + } unsigned long eviction_time_limit_us = evictionTimeLimitUs(); @@ -705,6 +711,16 @@ cant_free: latencyEndMonitor(latency); latencyAddSampleIfNeeded("eviction-cycle",latency); + +update_metrics: + if (result == EVICT_RUNNING || result == EVICT_FAIL) { + if (server.stat_last_eviction_exceeded_time == 0) + elapsedStart(&server.stat_last_eviction_exceeded_time); + } else if (result == EVICT_OK) { + if (server.stat_last_eviction_exceeded_time != 0) { + server.stat_total_eviction_exceeded_time += elapsedUs(server.stat_last_eviction_exceeded_time); + server.stat_last_eviction_exceeded_time = 0; + } + } return result; } - diff --git a/src/server.c b/src/server.c index aeed6584a..4d1a9ac5c 100644 --- a/src/server.c +++ b/src/server.c @@ -3092,6 +3092,8 @@ void resetServerStats(void) { server.stat_expired_time_cap_reached_count = 0; server.stat_expire_cycle_time_used = 0; server.stat_evictedkeys = 0; + server.stat_total_eviction_exceeded_time = 0; + server.stat_last_eviction_exceeded_time = 0; server.stat_keyspace_misses = 0; server.stat_keyspace_hits = 0; server.stat_active_defrag_hits = 0; @@ -4984,6 +4986,8 @@ sds genRedisInfoString(const char *section) { if (allsections || defsections || !strcasecmp(section,"stats")) { long long stat_total_reads_processed, stat_total_writes_processed; long long stat_net_input_bytes, stat_net_output_bytes; + long long current_eviction_exceeded_time = server.stat_last_eviction_exceeded_time ? + (long long) elapsedUs(server.stat_last_eviction_exceeded_time): 0; atomicGet(server.stat_total_reads_processed, stat_total_reads_processed); atomicGet(server.stat_total_writes_processed, stat_total_writes_processed); atomicGet(server.stat_net_input_bytes, stat_net_input_bytes); @@ -5008,6 +5012,8 @@ sds genRedisInfoString(const char *section) { "expired_time_cap_reached_count:%lld\r\n" "expire_cycle_cpu_milliseconds:%lld\r\n" "evicted_keys:%lld\r\n" + "total_eviction_exceeded_time:%lld\r\n" + "current_eviction_exceeded_time:%lld\r\n" "keyspace_hits:%lld\r\n" "keyspace_misses:%lld\r\n" "pubsub_channels:%ld\r\n" @@ -5046,6 +5052,8 @@ sds genRedisInfoString(const char *section) { server.stat_expired_time_cap_reached_count, server.stat_expire_cycle_time_used/1000, server.stat_evictedkeys, + (server.stat_total_eviction_exceeded_time + current_eviction_exceeded_time) / 1000, + current_eviction_exceeded_time / 1000, server.stat_keyspace_hits, server.stat_keyspace_misses, dictSize(server.pubsub_channels), diff --git a/src/server.h b/src/server.h index a7e2f49d2..cab2aaf12 100644 --- a/src/server.h +++ b/src/server.h @@ -1312,6 +1312,8 @@ struct redisServer { long long stat_expired_time_cap_reached_count; /* Early expire cycle stops.*/ long long stat_expire_cycle_time_used; /* Cumulative microseconds used. */ long long stat_evictedkeys; /* Number of evicted keys (maxmemory) */ + long long stat_total_eviction_exceeded_time; /* Total time over the memory limit, unit us */ + monotime stat_last_eviction_exceeded_time; /* Timestamp of current eviction start, unit us */ long long stat_keyspace_hits; /* Number of successful lookups of keys */ long long stat_keyspace_misses; /* Number of failed lookups of keys */ long long stat_active_defrag_hits; /* number of allocations moved */