From 49890c8ee9776f13aaabf7fe76d796a89de6bf1a Mon Sep 17 00:00:00 2001 From: Itamar Haber Date: Mon, 30 Apr 2018 19:33:01 +0300 Subject: [PATCH 1/2] Adds memory information about the script's cache to INFO MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implementation notes: as INFO is "already broken", I didn't want to break it further. Instead of computing the server.lua_script dict size on every call, I'm keeping a running sum of the body's length and dict overheads. This implementation is naive as it **does not** take into consideration dict rehashing, but that inaccuracy pays off in speed ;) Demo time: ```bash $ redis-cli info memory | grep "script" used_memory_scripts:96 used_memory_scripts_human:96B number_of_cached_scripts:0 $ redis-cli eval "" 0 ; redis-cli info memory | grep "script" (nil) used_memory_scripts:120 used_memory_scripts_human:120B number_of_cached_scripts:1 $ redis-cli script flush ; redis-cli info memory | grep "script" OK used_memory_scripts:96 used_memory_scripts_human:96B number_of_cached_scripts:0 $ redis-cli eval "return('Hello, Script Cache :)')" 0 ; redis-cli info memory | grep "script" "Hello, Script Cache :)" used_memory_scripts:152 used_memory_scripts_human:152B number_of_cached_scripts:1 $ redis-cli eval "return redis.sha1hex(\"return('Hello, Script Cache :)')\")" 0 ; redis-cli info memory | grep "script" "1be72729d43da5114929c1260a749073732dc822" used_memory_scripts:232 used_memory_scripts_human:232B number_of_cached_scripts:2 ✔ 19:03:54 redis [lua_scripts-in-info-memory L ✚…⚑] $ redis-cli evalsha 1be72729d43da5114929c1260a749073732dc822 0 "Hello, Script Cache :)" ``` --- src/scripting.c | 3 +++ src/server.c | 8 ++++++++ src/server.h | 1 + 3 files changed, 12 insertions(+) diff --git a/src/scripting.c b/src/scripting.c index 3c0597c7a..e131d8ae0 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -919,6 +919,7 @@ void scriptingInit(int setup) { * This is useful for replication, as we need to replicate EVALSHA * as EVAL, so we need to remember the associated script. */ server.lua_scripts = dictCreate(&shaScriptObjectDictType,NULL); + server.lua_scripts_mem = sizeof(dict); /* Register the redis commands table and fields */ lua_newtable(lua); @@ -1073,6 +1074,7 @@ void scriptingInit(int setup) { * This function is used in order to reset the scripting environment. */ void scriptingRelease(void) { dictRelease(server.lua_scripts); + server.lua_scripts_mem = 0; lua_close(server.lua); } @@ -1207,6 +1209,7 @@ sds luaCreateFunction(client *c, lua_State *lua, robj *body) { * EVALSHA commands as EVAL using the original script. */ int retval = dictAdd(server.lua_scripts,sha,body); serverAssertWithInfo(c ? c : server.lua_client,NULL,retval == DICT_OK); + server.lua_scripts_mem += sdslen(body->ptr) + sizeof(dictEntry); incrRefCount(body); return sha; } diff --git a/src/server.c b/src/server.c index 7f51778d7..9bab389c0 100644 --- a/src/server.c +++ b/src/server.c @@ -2994,6 +2994,7 @@ sds genRedisInfoString(char *section) { char peak_hmem[64]; char total_system_hmem[64]; char used_memory_lua_hmem[64]; + char used_memory_scripts_hmem[64]; char used_memory_rss_hmem[64]; char maxmemory_hmem[64]; size_t zmalloc_used = zmalloc_used_memory(); @@ -3013,6 +3014,7 @@ sds genRedisInfoString(char *section) { bytesToHuman(peak_hmem,server.stat_peak_memory); bytesToHuman(total_system_hmem,total_system_mem); bytesToHuman(used_memory_lua_hmem,memory_lua); + bytesToHuman(used_memory_scripts_hmem,server.lua_scripts_mem); bytesToHuman(used_memory_rss_hmem,server.cron_malloc_stats.process_rss); bytesToHuman(maxmemory_hmem,server.maxmemory); @@ -3037,6 +3039,9 @@ sds genRedisInfoString(char *section) { "total_system_memory_human:%s\r\n" "used_memory_lua:%lld\r\n" "used_memory_lua_human:%s\r\n" + "used_memory_scripts:%lld\r\n" + "used_memory_scripts_human:%s\r\n" + "number_of_cached_scripts:%lu\r\n" "maxmemory:%lld\r\n" "maxmemory_human:%s\r\n" "maxmemory_policy:%s\r\n" @@ -3069,6 +3074,9 @@ sds genRedisInfoString(char *section) { total_system_hmem, memory_lua, used_memory_lua_hmem, + server.lua_scripts_mem, + used_memory_scripts_hmem, + dictSize(server.lua_scripts), server.maxmemory, maxmemory_hmem, evict_policy, diff --git a/src/server.h b/src/server.h index 0e9c3f285..260ba80d7 100644 --- a/src/server.h +++ b/src/server.h @@ -1203,6 +1203,7 @@ struct redisServer { client *lua_client; /* The "fake client" to query Redis from Lua */ client *lua_caller; /* The client running EVAL right now, or NULL */ dict *lua_scripts; /* A dictionary of SHA1 -> Lua scripts */ + unsigned long long lua_scripts_mem; /* Cached scripts' memory + oh */ mstime_t lua_time_limit; /* Script timeout in milliseconds */ mstime_t lua_time_start; /* Start time of script, milliseconds time */ int lua_write_dirty; /* True if a write command was called during the From 993716c351856f30487d6c8d9cc64423183c5f1b Mon Sep 17 00:00:00 2001 From: Itamar Haber Date: Sun, 22 Jul 2018 21:16:00 +0300 Subject: [PATCH 2/2] Adds Lua overheads to MEMORY STATS, smartens the MEMORY DOCTOR --- src/object.c | 29 +++++++++++++++++++++++++++-- src/scripting.c | 4 ++-- src/server.h | 1 + 3 files changed, 30 insertions(+), 4 deletions(-) diff --git a/src/object.c b/src/object.c index 82b82632b..f72b29dd9 100644 --- a/src/object.c +++ b/src/object.c @@ -986,6 +986,18 @@ struct redisMemOverhead *getMemoryOverheadData(void) { mh->aof_buffer = mem; mem_total+=mem; + mem = 0; + mem += dictSize(server.lua_scripts) * sizeof(dictEntry) + + dictSlots(server.lua_scripts) * sizeof(dictEntry*); + mem += dictSize(server.repl_scriptcache_dict) * sizeof(dictEntry) + + dictSlots(server.repl_scriptcache_dict) * sizeof(dictEntry*); + if (listLength(server.repl_scriptcache_fifo) > 0) { + mem += listLength(server.repl_scriptcache_fifo) * (sizeof(listNode) + + sdsZmallocSize(listNodeValue(listFirst(server.repl_scriptcache_fifo)))); + } + mh->lua_caches = mem; + mem_total+=mem; + for (j = 0; j < server.dbnum; j++) { redisDb *db = server.db+j; long long keyscount = dictSize(db->dict); @@ -1043,6 +1055,7 @@ sds getMemoryDoctorReport(void) { int high_alloc_rss = 0; /* High rss overhead. */ int big_slave_buf = 0; /* Slave buffers are too big. */ int big_client_buf = 0; /* Client buffers are too big. */ + int many_scripts = 0; /* Script cache has too many scripts. */ int num_reports = 0; struct redisMemOverhead *mh = getMemoryOverheadData(); @@ -1093,6 +1106,12 @@ sds getMemoryDoctorReport(void) { big_slave_buf = 1; num_reports++; } + + /* Too many (over 42) scripts are cached? */ + if (dictSize(server.lua_scripts) > 42) { + many_scripts = 1; + num_reports++; + } } sds s; @@ -1122,7 +1141,7 @@ sds getMemoryDoctorReport(void) { s = sdscatprintf(s," * High allocator RSS overhead: This instance has an RSS memory overhead is greater than 1.1 (this means that the Resident Set Size of the allocator is much larger than the sum what the allocator actually holds). This problem is usually due to a large peak memory (check if there is a peak memory entry above in the report), you can try the MEMORY PURGE command to reclaim it.\n\n"); } if (high_proc_rss) { - s = sdscatprintf(s," * High process RSS overhead: This instance has non-allocator RSS memory overhead is greater than 1.1 (this means that the Resident Set Size of the Redis process is much larger than the RSS the allocator holds). This problem may be due to LUA scripts or Modules.\n\n"); + s = sdscatprintf(s," * High process RSS overhead: This instance has non-allocator RSS memory overhead is greater than 1.1 (this means that the Resident Set Size of the Redis process is much larger than the RSS the allocator holds). This problem may be due to Lua scripts or Modules.\n\n"); } if (big_slave_buf) { s = sdscat(s," * Big slave buffers: The slave output buffers in this instance are greater than 10MB for each slave (on average). This likely means that there is some slave instance that is struggling receiving data, either because it is too slow or because of networking issues. As a result, data piles on the master output buffers. Please try to identify what slave is not receiving data correctly and why. You can use the INFO output in order to check the slaves delays and the CLIENT LIST command to check the output buffers of each slave.\n\n"); @@ -1130,6 +1149,9 @@ sds getMemoryDoctorReport(void) { if (big_client_buf) { s = sdscat(s," * Big client buffers: The clients output buffers in this instance are greater than 200K per client (on average). This may result from different causes, like Pub/Sub clients subscribed to channels bot not receiving data fast enough, so that data piles on the Redis instance output buffer, or clients sending commands with large replies or very large sequences of commands in the same pipeline. Please use the CLIENT LIST command in order to investigate the issue if it causes problems in your instance, or to understand better why certain clients are using a big amount of memory.\n\n"); } + if (many_scripts) { + s = sdscat(s," * Many scripts: There seem to be many cached scripts in this instance (more than 42). This may be because scripts are generated and `EVAL`ed, instead of being parameterized (with KEYS and ARGV), `SCRIPT LOAD`ed and `EVALSHA`ed. Unless `SCRIPT FLUSH` is called periodically, the scripts' caches may end up consuming most of your memory.\n\n"); + } s = sdscat(s,"I'm here to keep you safe, Sam. I want to help you.\n"); } freeMemoryOverheadData(mh); @@ -1236,7 +1258,7 @@ void memoryCommand(client *c) { } else if (!strcasecmp(c->argv[1]->ptr,"stats") && c->argc == 2) { struct redisMemOverhead *mh = getMemoryOverheadData(); - addReplyMultiBulkLen(c,(24+mh->num_dbs)*2); + addReplyMultiBulkLen(c,(25+mh->num_dbs)*2); addReplyBulkCString(c,"peak.allocated"); addReplyLongLong(c,mh->peak_allocated); @@ -1259,6 +1281,9 @@ void memoryCommand(client *c) { addReplyBulkCString(c,"aof.buffer"); addReplyLongLong(c,mh->aof_buffer); + addReplyBulkCString(c,"lua.caches"); + addReplyLongLong(c,mh->lua_caches); + for (size_t j = 0; j < mh->num_dbs; j++) { char dbname[32]; snprintf(dbname,sizeof(dbname),"db.%zd",mh->db[j].dbid); diff --git a/src/scripting.c b/src/scripting.c index e131d8ae0..1a9cc1c60 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -919,7 +919,7 @@ void scriptingInit(int setup) { * This is useful for replication, as we need to replicate EVALSHA * as EVAL, so we need to remember the associated script. */ server.lua_scripts = dictCreate(&shaScriptObjectDictType,NULL); - server.lua_scripts_mem = sizeof(dict); + server.lua_scripts_mem = 0; /* Register the redis commands table and fields */ lua_newtable(lua); @@ -1209,7 +1209,7 @@ sds luaCreateFunction(client *c, lua_State *lua, robj *body) { * EVALSHA commands as EVAL using the original script. */ int retval = dictAdd(server.lua_scripts,sha,body); serverAssertWithInfo(c ? c : server.lua_client,NULL,retval == DICT_OK); - server.lua_scripts_mem += sdslen(body->ptr) + sizeof(dictEntry); + server.lua_scripts_mem += sdsZmallocSize(sha) + sdsZmallocSize(body->ptr); incrRefCount(body); return sha; } diff --git a/src/server.h b/src/server.h index 260ba80d7..3d9fa2307 100644 --- a/src/server.h +++ b/src/server.h @@ -834,6 +834,7 @@ struct redisMemOverhead { size_t clients_slaves; size_t clients_normal; size_t aof_buffer; + size_t lua_caches; size_t overhead_total; size_t dataset; size_t total_keys;