gvsafronov/futriix
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Introduced a new log verbosity level, so now DEBUG is really for debugging. Refactored a bit maxmemory. When virtual memory is short in RAM free the objects freelist as well as swapping things out.

Browse Source
This commit is contained in:
antirez 2010-01-06 09:15:17 -05:00
parent 38823f0817
commit f870935d3e
3 changed files with 51 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
TODO
View File

@ -11,6 +11,13 @@ VERSION 1.4 TODO (Hash type)
* Synchronous Virtual Memory * Synchronous Virtual Memory
* BLPOP & C. tests (write a non blocking Tcl client as first step) * BLPOP & C. tests (write a non blocking Tcl client as first step)
Virtual Memory sub-TODO:
* Check if the page selection algorithm is working well.
* Fix support for large files
* Divide swappability of objects by refcount
* While loading DB from snapshot or AOF, swap objects as needed if maxmemory
is reached, calling swapOneObject().
VERSION 1.6 TODO (Virtual memory) VERSION 1.6 TODO (Virtual memory)
================================= =================================

73
redis.c
View File

@ -197,8 +197,9 @@
/* Log levels */ /* Log levels */
#define REDIS_DEBUG 0 #define REDIS_DEBUG 0
#define REDIS_NOTICE 1 #define REDIS_VERBOSE 1
#define REDIS_WARNING 2 #define REDIS_NOTICE 2
#define REDIS_WARNING 3
/* Anti-warning macro... */ /* Anti-warning macro... */
#define REDIS_NOTUSED(V) ((void) V) #define REDIS_NOTUSED(V) ((void) V)
@ -496,6 +497,7 @@ static robj *vmLoadObject(robj *key);
static robj *vmPreviewObject(robj *key); static robj *vmPreviewObject(robj *key);
static int vmSwapOneObject(void); static int vmSwapOneObject(void);
static int vmCanSwapOut(void); static int vmCanSwapOut(void);
static void freeOneObjectFromFreelist(void);
static void authCommand(redisClient *c); static void authCommand(redisClient *c);
static void pingCommand(redisClient *c); static void pingCommand(redisClient *c);
@ -949,7 +951,7 @@ static void closeTimedoutClients(void) {
!(c->flags & REDIS_MASTER) && /* no timeout for masters */ !(c->flags & REDIS_MASTER) && /* no timeout for masters */
(now - c->lastinteraction > server.maxidletime)) (now - c->lastinteraction > server.maxidletime))
{ {
redisLog(REDIS_DEBUG,"Closing idle client"); redisLog(REDIS_VERBOSE,"Closing idle client");
freeClient(c); freeClient(c);
} else if (c->flags & REDIS_BLOCKED) { } else if (c->flags & REDIS_BLOCKED) {
if (c->blockingto != 0 && c->blockingto < now) { if (c->blockingto != 0 && c->blockingto < now) {
@ -976,9 +978,9 @@ static void tryResizeHashTables(void) {
for (j = 0; j < server.dbnum; j++) { for (j = 0; j < server.dbnum; j++) {
if (htNeedsResize(server.db[j].dict)) { if (htNeedsResize(server.db[j].dict)) {
redisLog(REDIS_DEBUG,"The hash table %d is too sparse, resize it...",j); redisLog(REDIS_VERBOSE,"The hash table %d is too sparse, resize it...",j);
dictResize(server.db[j].dict); dictResize(server.db[j].dict);
redisLog(REDIS_DEBUG,"Hash table %d resized.",j); redisLog(REDIS_VERBOSE,"Hash table %d resized.",j);
} }
if (htNeedsResize(server.db[j].expires)) if (htNeedsResize(server.db[j].expires))
dictResize(server.db[j].expires); dictResize(server.db[j].expires);
@ -1092,7 +1094,7 @@ static int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientD
used = dictSize(server.db[j].dict); used = dictSize(server.db[j].dict);
vkeys = dictSize(server.db[j].expires); vkeys = dictSize(server.db[j].expires);
if (!(loops % 5) && (used || vkeys)) { if (!(loops % 5) && (used || vkeys)) {
redisLog(REDIS_DEBUG,"DB %d: %lld keys (%lld volatile) in %lld slots HT.",j,used,vkeys,size); redisLog(REDIS_VERBOSE,"DB %d: %lld keys (%lld volatile) in %lld slots HT.",j,used,vkeys,size);
/* dictPrintStats(server.dict); */ /* dictPrintStats(server.dict); */
} }
} }
@ -1107,7 +1109,7 @@ static int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientD
/* Show information about connected clients */ /* Show information about connected clients */
if (!(loops % 5)) { if (!(loops % 5)) {
redisLog(REDIS_DEBUG,"%d clients connected (%d slaves), %zu bytes in use, %d shared objects", redisLog(REDIS_VERBOSE,"%d clients connected (%d slaves), %zu bytes in use, %d shared objects",
listLength(server.clients)-listLength(server.slaves), listLength(server.clients)-listLength(server.slaves),
listLength(server.slaves), listLength(server.slaves),
server.usedmemory, server.usedmemory,
@ -1179,12 +1181,15 @@ static int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientD
} }
/* Swap a few keys on disk if we are over the memory limit and VM /* Swap a few keys on disk if we are over the memory limit and VM
* is enbled. */ * is enbled. Try to free objects from the free list first. */
if (vmCanSwapOut()) { if (vmCanSwapOut()) {
while (server.vm_enabled && zmalloc_used_memory() > while (server.vm_enabled && zmalloc_used_memory() >
server.vm_max_memory) { server.vm_max_memory)
if (vmSwapOneObject() == REDIS_ERR) { {
if (zmalloc_used_memory() > if (listLength(server.objfreelist)) {
freeOneObjectFromFreelist();
} else if (vmSwapOneObject() == REDIS_ERR) {
if ((loops % 30) == 0 && zmalloc_used_memory() >
(server.vm_max_memory+server.vm_max_memory/10)) { (server.vm_max_memory+server.vm_max_memory/10)) {
redisLog(REDIS_WARNING,"WARNING: vm-max-memory limit exceeded by more than 10%% but unable to swap more objects out!"); redisLog(REDIS_WARNING,"WARNING: vm-max-memory limit exceeded by more than 10%% but unable to swap more objects out!");
} }
@ -1256,7 +1261,7 @@ static void resetServerSaveParams() {
static void initServerConfig() { static void initServerConfig() {
server.dbnum = REDIS_DEFAULT_DBNUM; server.dbnum = REDIS_DEFAULT_DBNUM;
server.port = REDIS_SERVERPORT; server.port = REDIS_SERVERPORT;
server.verbosity = REDIS_DEBUG; server.verbosity = REDIS_VERBOSE;
server.maxidletime = REDIS_MAXIDLETIME; server.maxidletime = REDIS_MAXIDLETIME;
server.saveparams = NULL; server.saveparams = NULL;
server.logfile = NULL; /* NULL = log on standard output */ server.logfile = NULL; /* NULL = log on standard output */
@ -1436,6 +1441,7 @@ static void loadServerConfig(char *filename) {
} }
} else if (!strcasecmp(argv[0],"loglevel") && argc == 2) { } else if (!strcasecmp(argv[0],"loglevel") && argc == 2) {
if (!strcasecmp(argv[1],"debug")) server.verbosity = REDIS_DEBUG; if (!strcasecmp(argv[1],"debug")) server.verbosity = REDIS_DEBUG;
else if (!strcasecmp(argv[1],"verbose")) server.verbosity = REDIS_VERBOSE;
else if (!strcasecmp(argv[1],"notice")) server.verbosity = REDIS_NOTICE; else if (!strcasecmp(argv[1],"notice")) server.verbosity = REDIS_NOTICE;
else if (!strcasecmp(argv[1],"warning")) server.verbosity = REDIS_WARNING; else if (!strcasecmp(argv[1],"warning")) server.verbosity = REDIS_WARNING;
else { else {
@ -1677,7 +1683,7 @@ static void sendReplyToClient(aeEventLoop *el, int fd, void *privdata, int mask)
if (errno == EAGAIN) { if (errno == EAGAIN) {
nwritten = 0; nwritten = 0;
} else { } else {
redisLog(REDIS_DEBUG, redisLog(REDIS_VERBOSE,
"Error writing to client: %s", strerror(errno)); "Error writing to client: %s", strerror(errno));
freeClient(c); freeClient(c);
return; return;
@ -1730,7 +1736,7 @@ static void sendReplyToClientWritev(aeEventLoop *el, int fd, void *privdata, int
/* write all collected blocks at once */ /* write all collected blocks at once */
if((nwritten = writev(fd, iov, ion)) < 0) { if((nwritten = writev(fd, iov, ion)) < 0) {
if (errno != EAGAIN) { if (errno != EAGAIN) {
redisLog(REDIS_DEBUG, redisLog(REDIS_VERBOSE,
"Error writing to client: %s", strerror(errno)); "Error writing to client: %s", strerror(errno));
freeClient(c); freeClient(c);
return; return;
@ -2091,7 +2097,7 @@ again:
} }
return; return;
} else if (sdslen(c->querybuf) >= REDIS_REQUEST_MAX_SIZE) { } else if (sdslen(c->querybuf) >= REDIS_REQUEST_MAX_SIZE) {
redisLog(REDIS_DEBUG, "Client protocol error"); redisLog(REDIS_VERBOSE, "Client protocol error");
freeClient(c); freeClient(c);
return; return;
} }
@ -2128,12 +2134,12 @@ static void readQueryFromClient(aeEventLoop *el, int fd, void *privdata, int mas
if (errno == EAGAIN) { if (errno == EAGAIN) {
nread = 0; nread = 0;
} else { } else {
redisLog(REDIS_DEBUG, "Reading from client: %s",strerror(errno)); redisLog(REDIS_VERBOSE, "Reading from client: %s",strerror(errno));
freeClient(c); freeClient(c);
return; return;
} }
} else if (nread == 0) { } else if (nread == 0) {
redisLog(REDIS_DEBUG, "Client closed connection"); redisLog(REDIS_VERBOSE, "Client closed connection");
freeClient(c); freeClient(c);
return; return;
} }
@ -2252,10 +2258,10 @@ static void acceptHandler(aeEventLoop *el, int fd, void *privdata, int mask) {
cfd = anetAccept(server.neterr, fd, cip, &cport); cfd = anetAccept(server.neterr, fd, cip, &cport);
if (cfd == AE_ERR) { if (cfd == AE_ERR) {
redisLog(REDIS_DEBUG,"Accepting client connection: %s", server.neterr); redisLog(REDIS_VERBOSE,"Accepting client connection: %s", server.neterr);
return; return;
} }
redisLog(REDIS_DEBUG,"Accepted %s:%d", cip, cport); redisLog(REDIS_VERBOSE,"Accepted %s:%d", cip, cport);
if ((c = createClient(cfd)) == NULL) { if ((c = createClient(cfd)) == NULL) {
redisLog(REDIS_WARNING,"Error allocating resoures for the client"); redisLog(REDIS_WARNING,"Error allocating resoures for the client");
close(cfd); /* May be already closed, just ingore errors */ close(cfd); /* May be already closed, just ingore errors */
@ -6002,7 +6008,7 @@ static void sendBulkToSlave(aeEventLoop *el, int fd, void *privdata, int mask) {
return; return;
} }
if ((nwritten = write(fd,buf,buflen)) == -1) { if ((nwritten = write(fd,buf,buflen)) == -1) {
redisLog(REDIS_DEBUG,"Write error sending DB to slave: %s", redisLog(REDIS_VERBOSE,"Write error sending DB to slave: %s",
strerror(errno)); strerror(errno));
freeClient(slave); freeClient(slave);
return; return;
@ -6206,6 +6212,18 @@ static void slaveofCommand(redisClient *c) {
/* ============================ Maxmemory directive ======================== */ /* ============================ Maxmemory directive ======================== */
/* Free one object form the pre-allocated objects free list. This is useful
* under low mem conditions as by default we take 1 million free objects
* allocated. */
static void freeOneObjectFromFreelist(void) {
robj *o;
listNode *head = listFirst(server.objfreelist);
o = listNodeValue(head);
listDelNode(server.objfreelist,head);
zfree(o);
}
/* This function gets called when 'maxmemory' is set on the config file to limit /* This function gets called when 'maxmemory' is set on the config file to limit
* the max memory used by the server, and we are out of memory. * the max memory used by the server, and we are out of memory.
* This function will try to, in order: * This function will try to, in order:
@ -6220,12 +6238,7 @@ static void slaveofCommand(redisClient *c) {
static void freeMemoryIfNeeded(void) { static void freeMemoryIfNeeded(void) {
while (server.maxmemory && zmalloc_used_memory() > server.maxmemory) { while (server.maxmemory && zmalloc_used_memory() > server.maxmemory) {
if (listLength(server.objfreelist)) { if (listLength(server.objfreelist)) {
robj *o; freeOneObjectFromFreelist();
listNode *head = listFirst(server.objfreelist);
o = listNodeValue(head);
listDelNode(server.objfreelist,head);
zfree(o);
} else { } else {
int j, k, freed = 0; int j, k, freed = 0;
@ -6731,7 +6744,7 @@ static void vmInit(void) {
redisLog(REDIS_NOTICE,"Swap file allocated with success"); redisLog(REDIS_NOTICE,"Swap file allocated with success");
} }
server.vm_bitmap = zmalloc((server.vm_pages+7)/8); server.vm_bitmap = zmalloc((server.vm_pages+7)/8);
redisLog(REDIS_DEBUG,"Allocated %lld bytes page table for %lld pages", redisLog(REDIS_VERBOSE,"Allocated %lld bytes page table for %lld pages",
(long long) (server.vm_pages+7)/8, server.vm_pages); (long long) (server.vm_pages+7)/8, server.vm_pages);
memset(server.vm_bitmap,0,(server.vm_pages+7)/8); memset(server.vm_bitmap,0,(server.vm_pages+7)/8);
/* Try to remove the swap file, so the OS will really delete it from the /* Try to remove the swap file, so the OS will really delete it from the
@ -6744,8 +6757,8 @@ static void vmMarkPageUsed(off_t page) {
off_t byte = page/8; off_t byte = page/8;
int bit = page&7; int bit = page&7;
server.vm_bitmap[byte] |= 1<<bit; server.vm_bitmap[byte] |= 1<<bit;
printf("Mark used: %lld (byte:%lld bit:%d)\n", (long long)page, redisLog(REDIS_DEBUG,"Mark used: %lld (byte:%lld bit:%d)\n",
(long long)byte, bit); (long long)page, (long long)byte, bit);
} }
/* Mark N contiguous pages as used, with 'page' being the first. */ /* Mark N contiguous pages as used, with 'page' being the first. */
@ -6810,7 +6823,7 @@ static int vmFindContiguousPages(off_t *first, int n) {
while(offset < server.vm_pages) { while(offset < server.vm_pages) {
off_t this = base+offset; off_t this = base+offset;
printf("THIS: %lld (%c)\n", (long long) this, vmFreePage(this) ? 'F' : 'X'); redisLog(REDIS_DEBUG, "THIS: %lld (%c)\n", (long long) this, vmFreePage(this) ? 'F' : 'X');
/* If we overflow, restart from page zero */ /* If we overflow, restart from page zero */
if (this >= server.vm_pages) { if (this >= server.vm_pages) {
this -= server.vm_pages; this -= server.vm_pages;

2
redis.conf
View File

@ -24,7 +24,7 @@ timeout 300
# debug (a lot of information, useful for development/testing) # debug (a lot of information, useful for development/testing)
# notice (moderately verbose, what you want in production probably) # notice (moderately verbose, what you want in production probably)
# warning (only very important / critical messages are logged) # warning (only very important / critical messages are logged)
loglevel debug loglevel verbose
# Specify the log file name. Also 'stdout' can be used to force # Specify the log file name. Also 'stdout' can be used to force
# the demon to log on the standard output. Note that if you use standard # the demon to log on the standard output. Note that if you use standard