mirror of https://mirror.osredm.com/root/redis.git
Add Lua VM memory to memory overhead, now that it's part of zmalloc (#13660)
To complement the work done in #13133. it added the script VMs memory to be counted as part of zmalloc, but that means they should be also counted as part of the non-value overhead. this commit contains some refactoring to make variable names and function names less confusing. it also adds a new field named `script.VMs` into the `MEMORY STATS` command. additionally, clear scripts and stats between tests in external mode (which is related to how this issue was discovered)
This commit is contained in:
parent
5b84dc9678
commit
79fd255828
|
@ -44,6 +44,9 @@
|
||||||
"lua.caches": {
|
"lua.caches": {
|
||||||
"type": "integer"
|
"type": "integer"
|
||||||
},
|
},
|
||||||
|
"script.VMs": {
|
||||||
|
"type": "integer"
|
||||||
|
},
|
||||||
"functions.caches": {
|
"functions.caches": {
|
||||||
"type": "integer"
|
"type": "integer"
|
||||||
},
|
},
|
||||||
|
|
|
@ -730,7 +730,7 @@ NULL
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
unsigned long evalMemory(void) {
|
unsigned long evalScriptsMemoryVM(void) {
|
||||||
return luaMemory(lctx.lua);
|
return luaMemory(lctx.lua);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -738,7 +738,7 @@ dict* evalScriptsDict(void) {
|
||||||
return lctx.lua_scripts;
|
return lctx.lua_scripts;
|
||||||
}
|
}
|
||||||
|
|
||||||
unsigned long evalScriptsMemory(void) {
|
unsigned long evalScriptsMemoryEngine(void) {
|
||||||
return lctx.lua_scripts_mem +
|
return lctx.lua_scripts_mem +
|
||||||
dictMemUsage(lctx.lua_scripts) +
|
dictMemUsage(lctx.lua_scripts) +
|
||||||
dictSize(lctx.lua_scripts) * sizeof(luaScript) +
|
dictSize(lctx.lua_scripts) * sizeof(luaScript) +
|
||||||
|
|
|
@ -1063,7 +1063,7 @@ void functionLoadCommand(client *c) {
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Return memory usage of all the engines combine */
|
/* Return memory usage of all the engines combine */
|
||||||
unsigned long functionsMemory(void) {
|
unsigned long functionsMemoryVM(void) {
|
||||||
dictIterator *iter = dictGetIterator(engines);
|
dictIterator *iter = dictGetIterator(engines);
|
||||||
dictEntry *entry = NULL;
|
dictEntry *entry = NULL;
|
||||||
size_t engines_memory = 0;
|
size_t engines_memory = 0;
|
||||||
|
@ -1078,7 +1078,7 @@ unsigned long functionsMemory(void) {
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Return memory overhead of all the engines combine */
|
/* Return memory overhead of all the engines combine */
|
||||||
unsigned long functionsMemoryOverhead(void) {
|
unsigned long functionsMemoryEngine(void) {
|
||||||
size_t memory_overhead = dictMemUsage(engines);
|
size_t memory_overhead = dictMemUsage(engines);
|
||||||
memory_overhead += dictMemUsage(curr_functions_lib_ctx->functions);
|
memory_overhead += dictMemUsage(curr_functions_lib_ctx->functions);
|
||||||
memory_overhead += sizeof(functionsLibCtx);
|
memory_overhead += sizeof(functionsLibCtx);
|
||||||
|
|
|
@ -102,8 +102,8 @@ struct functionLibInfo {
|
||||||
|
|
||||||
int functionsRegisterEngine(const char *engine_name, engine *engine_ctx);
|
int functionsRegisterEngine(const char *engine_name, engine *engine_ctx);
|
||||||
sds functionsCreateWithLibraryCtx(sds code, int replace, sds* err, functionsLibCtx *lib_ctx, size_t timeout);
|
sds functionsCreateWithLibraryCtx(sds code, int replace, sds* err, functionsLibCtx *lib_ctx, size_t timeout);
|
||||||
unsigned long functionsMemory(void);
|
unsigned long functionsMemoryVM(void);
|
||||||
unsigned long functionsMemoryOverhead(void);
|
unsigned long functionsMemoryEngine(void);
|
||||||
unsigned long functionsNum(void);
|
unsigned long functionsNum(void);
|
||||||
unsigned long functionsLibNum(void);
|
unsigned long functionsLibNum(void);
|
||||||
dict* functionsLibGet(void);
|
dict* functionsLibGet(void);
|
||||||
|
|
17
src/object.c
17
src/object.c
|
@ -1230,12 +1230,16 @@ struct redisMemOverhead *getMemoryOverheadData(void) {
|
||||||
mh->aof_buffer = mem;
|
mh->aof_buffer = mem;
|
||||||
mem_total+=mem;
|
mem_total+=mem;
|
||||||
|
|
||||||
mem = evalScriptsMemory();
|
mem = evalScriptsMemoryEngine();
|
||||||
mh->lua_caches = mem;
|
mh->eval_caches = mem;
|
||||||
mem_total+=mem;
|
mem_total+=mem;
|
||||||
mh->functions_caches = functionsMemoryOverhead();
|
mh->functions_caches = functionsMemoryEngine();
|
||||||
mem_total+=mh->functions_caches;
|
mem_total+=mh->functions_caches;
|
||||||
|
|
||||||
|
mh->script_vm = evalScriptsMemoryVM();
|
||||||
|
mh->script_vm += functionsMemoryVM();
|
||||||
|
mem_total+=mh->script_vm;
|
||||||
|
|
||||||
for (j = 0; j < server.dbnum; j++) {
|
for (j = 0; j < server.dbnum; j++) {
|
||||||
redisDb *db = server.db+j;
|
redisDb *db = server.db+j;
|
||||||
if (!kvstoreNumAllocatedDicts(db->keys)) continue;
|
if (!kvstoreNumAllocatedDicts(db->keys)) continue;
|
||||||
|
@ -1556,7 +1560,7 @@ NULL
|
||||||
} else if (!strcasecmp(c->argv[1]->ptr,"stats") && c->argc == 2) {
|
} else if (!strcasecmp(c->argv[1]->ptr,"stats") && c->argc == 2) {
|
||||||
struct redisMemOverhead *mh = getMemoryOverheadData();
|
struct redisMemOverhead *mh = getMemoryOverheadData();
|
||||||
|
|
||||||
addReplyMapLen(c,31+mh->num_dbs);
|
addReplyMapLen(c,32+mh->num_dbs);
|
||||||
|
|
||||||
addReplyBulkCString(c,"peak.allocated");
|
addReplyBulkCString(c,"peak.allocated");
|
||||||
addReplyLongLong(c,mh->peak_allocated);
|
addReplyLongLong(c,mh->peak_allocated);
|
||||||
|
@ -1583,11 +1587,14 @@ NULL
|
||||||
addReplyLongLong(c,mh->aof_buffer);
|
addReplyLongLong(c,mh->aof_buffer);
|
||||||
|
|
||||||
addReplyBulkCString(c,"lua.caches");
|
addReplyBulkCString(c,"lua.caches");
|
||||||
addReplyLongLong(c,mh->lua_caches);
|
addReplyLongLong(c,mh->eval_caches);
|
||||||
|
|
||||||
addReplyBulkCString(c,"functions.caches");
|
addReplyBulkCString(c,"functions.caches");
|
||||||
addReplyLongLong(c,mh->functions_caches);
|
addReplyLongLong(c,mh->functions_caches);
|
||||||
|
|
||||||
|
addReplyBulkCString(c,"script.VMs");
|
||||||
|
addReplyLongLong(c,mh->script_vm);
|
||||||
|
|
||||||
for (size_t j = 0; j < mh->num_dbs; j++) {
|
for (size_t j = 0; j < mh->num_dbs; j++) {
|
||||||
char dbname[32];
|
char dbname[32];
|
||||||
snprintf(dbname,sizeof(dbname),"db.%zd",mh->db[j].dbid);
|
snprintf(dbname,sizeof(dbname),"db.%zd",mh->db[j].dbid);
|
||||||
|
|
10
src/server.c
10
src/server.c
|
@ -5700,8 +5700,8 @@ sds genRedisInfoString(dict *section_dict, int all_sections, int everything) {
|
||||||
size_t zmalloc_used = zmalloc_used_memory();
|
size_t zmalloc_used = zmalloc_used_memory();
|
||||||
size_t total_system_mem = server.system_memory_size;
|
size_t total_system_mem = server.system_memory_size;
|
||||||
const char *evict_policy = evictPolicyToString();
|
const char *evict_policy = evictPolicyToString();
|
||||||
long long memory_lua = evalMemory();
|
long long memory_lua = evalScriptsMemoryVM();
|
||||||
long long memory_functions = functionsMemory();
|
long long memory_functions = functionsMemoryVM();
|
||||||
struct redisMemOverhead *mh = getMemoryOverheadData();
|
struct redisMemOverhead *mh = getMemoryOverheadData();
|
||||||
|
|
||||||
/* Peak memory is updated from time to time by serverCron() so it
|
/* Peak memory is updated from time to time by serverCron() so it
|
||||||
|
@ -5716,7 +5716,7 @@ sds genRedisInfoString(dict *section_dict, int all_sections, int everything) {
|
||||||
bytesToHuman(total_system_hmem,sizeof(total_system_hmem),total_system_mem);
|
bytesToHuman(total_system_hmem,sizeof(total_system_hmem),total_system_mem);
|
||||||
bytesToHuman(used_memory_lua_hmem,sizeof(used_memory_lua_hmem),memory_lua);
|
bytesToHuman(used_memory_lua_hmem,sizeof(used_memory_lua_hmem),memory_lua);
|
||||||
bytesToHuman(used_memory_vm_total_hmem,sizeof(used_memory_vm_total_hmem),memory_functions + memory_lua);
|
bytesToHuman(used_memory_vm_total_hmem,sizeof(used_memory_vm_total_hmem),memory_functions + memory_lua);
|
||||||
bytesToHuman(used_memory_scripts_hmem,sizeof(used_memory_scripts_hmem),mh->lua_caches + mh->functions_caches);
|
bytesToHuman(used_memory_scripts_hmem,sizeof(used_memory_scripts_hmem),mh->eval_caches + mh->functions_caches);
|
||||||
bytesToHuman(used_memory_rss_hmem,sizeof(used_memory_rss_hmem),server.cron_malloc_stats.process_rss);
|
bytesToHuman(used_memory_rss_hmem,sizeof(used_memory_rss_hmem),server.cron_malloc_stats.process_rss);
|
||||||
bytesToHuman(maxmemory_hmem,sizeof(maxmemory_hmem),server.maxmemory);
|
bytesToHuman(maxmemory_hmem,sizeof(maxmemory_hmem),server.maxmemory);
|
||||||
|
|
||||||
|
@ -5742,7 +5742,7 @@ sds genRedisInfoString(dict *section_dict, int all_sections, int everything) {
|
||||||
"used_memory_lua:%lld\r\n", memory_lua, /* deprecated, renamed to used_memory_vm_eval */
|
"used_memory_lua:%lld\r\n", memory_lua, /* deprecated, renamed to used_memory_vm_eval */
|
||||||
"used_memory_vm_eval:%lld\r\n", memory_lua,
|
"used_memory_vm_eval:%lld\r\n", memory_lua,
|
||||||
"used_memory_lua_human:%s\r\n", used_memory_lua_hmem, /* deprecated */
|
"used_memory_lua_human:%s\r\n", used_memory_lua_hmem, /* deprecated */
|
||||||
"used_memory_scripts_eval:%lld\r\n", (long long)mh->lua_caches,
|
"used_memory_scripts_eval:%lld\r\n", (long long)mh->eval_caches,
|
||||||
"number_of_cached_scripts:%lu\r\n", dictSize(evalScriptsDict()),
|
"number_of_cached_scripts:%lu\r\n", dictSize(evalScriptsDict()),
|
||||||
"number_of_functions:%lu\r\n", functionsNum(),
|
"number_of_functions:%lu\r\n", functionsNum(),
|
||||||
"number_of_libraries:%lu\r\n", functionsLibNum(),
|
"number_of_libraries:%lu\r\n", functionsLibNum(),
|
||||||
|
@ -5750,7 +5750,7 @@ sds genRedisInfoString(dict *section_dict, int all_sections, int everything) {
|
||||||
"used_memory_vm_total:%lld\r\n", memory_functions + memory_lua,
|
"used_memory_vm_total:%lld\r\n", memory_functions + memory_lua,
|
||||||
"used_memory_vm_total_human:%s\r\n", used_memory_vm_total_hmem,
|
"used_memory_vm_total_human:%s\r\n", used_memory_vm_total_hmem,
|
||||||
"used_memory_functions:%lld\r\n", (long long)mh->functions_caches,
|
"used_memory_functions:%lld\r\n", (long long)mh->functions_caches,
|
||||||
"used_memory_scripts:%lld\r\n", (long long)mh->lua_caches + (long long)mh->functions_caches,
|
"used_memory_scripts:%lld\r\n", (long long)mh->eval_caches + (long long)mh->functions_caches,
|
||||||
"used_memory_scripts_human:%s\r\n", used_memory_scripts_hmem,
|
"used_memory_scripts_human:%s\r\n", used_memory_scripts_hmem,
|
||||||
"maxmemory:%lld\r\n", server.maxmemory,
|
"maxmemory:%lld\r\n", server.maxmemory,
|
||||||
"maxmemory_human:%s\r\n", maxmemory_hmem,
|
"maxmemory_human:%s\r\n", maxmemory_hmem,
|
||||||
|
|
|
@ -1401,8 +1401,9 @@ struct redisMemOverhead {
|
||||||
size_t clients_normal;
|
size_t clients_normal;
|
||||||
size_t cluster_links;
|
size_t cluster_links;
|
||||||
size_t aof_buffer;
|
size_t aof_buffer;
|
||||||
size_t lua_caches;
|
size_t eval_caches;
|
||||||
size_t functions_caches;
|
size_t functions_caches;
|
||||||
|
size_t script_vm;
|
||||||
size_t overhead_total;
|
size_t overhead_total;
|
||||||
size_t dataset;
|
size_t dataset;
|
||||||
size_t total_keys;
|
size_t total_keys;
|
||||||
|
@ -3510,9 +3511,9 @@ int ldbIsEnabled(void);
|
||||||
void ldbLog(sds entry);
|
void ldbLog(sds entry);
|
||||||
void ldbLogRedisReply(char *reply);
|
void ldbLogRedisReply(char *reply);
|
||||||
void sha1hex(char *digest, char *script, size_t len);
|
void sha1hex(char *digest, char *script, size_t len);
|
||||||
unsigned long evalMemory(void);
|
unsigned long evalScriptsMemoryVM(void);
|
||||||
dict* evalScriptsDict(void);
|
dict* evalScriptsDict(void);
|
||||||
unsigned long evalScriptsMemory(void);
|
unsigned long evalScriptsMemoryEngine(void);
|
||||||
uint64_t evalGetCommandFlags(client *c, uint64_t orig_flags);
|
uint64_t evalGetCommandFlags(client *c, uint64_t orig_flags);
|
||||||
uint64_t fcallGetCommandFlags(client *c, uint64_t orig_flags);
|
uint64_t fcallGetCommandFlags(client *c, uint64_t orig_flags);
|
||||||
int isInsideYieldingLongCommand(void);
|
int isInsideYieldingLongCommand(void);
|
||||||
|
|
|
@ -373,6 +373,8 @@ proc run_external_server_test {code overrides} {
|
||||||
|
|
||||||
r flushall
|
r flushall
|
||||||
r function flush
|
r function flush
|
||||||
|
r script flush
|
||||||
|
r config resetstat
|
||||||
|
|
||||||
# store configs
|
# store configs
|
||||||
set saved_config {}
|
set saved_config {}
|
||||||
|
|
Loading…
Reference in New Issue