gvsafronov/futriix
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Change default back to jemalloc (until defrag is implemented) and fix compile errors

Browse Source
This commit is contained in:
John Sully 2019-02-21 01:36:38 -05:00
parent 75f589c702
commit 41164fbb61
7 changed files with 62 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
src/Makefile
View File

@ -39,7 +39,7 @@ MALLOC=libc
ifneq ($(uname_M),armv6l)
ifneq ($(uname_M),armv7l)
ifeq ($(uname_S),Linux)
MALLOC=memkind
MALLOC=jemalloc
endif
endif
endif
@ -134,23 +134,27 @@ FINAL_CXXFLAGS+= -I../deps/hiredis -I../deps/linenoise -I../deps/lua/src
ifeq ($(MALLOC),tcmalloc)
FINAL_CFLAGS+= -DUSE_TCMALLOC
FINAL_CXXFLAGS+= -DUSE_TCMALLOC
FINAL_LIBS+= -ltcmalloc
endif
ifeq ($(MALLOC),tcmalloc_minimal)
FINAL_CFLAGS+= -DUSE_TCMALLOC
FINAL_CXXFLAGS+= -DUSE_TCMALLOC
FINAL_LIBS+= -ltcmalloc_minimal
endif
ifeq ($(MALLOC),jemalloc)
DEPENDENCY_TARGETS+= jemalloc
FINAL_CFLAGS+= -DUSE_JEMALLOC -I../deps/jemalloc/include
FINAL_CXXFLAGS+= -DUSE_JEMALLOC -I../deps/jemalloc/include
FINAL_LIBS := ../deps/jemalloc/lib/libjemalloc.a $(FINAL_LIBS)
endif
ifeq ($(MALLOC),memkind)
DEPENDENCY_TARGETS+= memkind
FINAL_CFLAGS+= -DUSE_MEMKIND -I../deps/memkind/src/include
FINAL_CXXFLAGS+= -DUSE_MEMKIND -I../deps/memkind/src/include
FINAL_LIBS := ../deps/memkind/src/.libs/libmemkind.a -lnuma $(FINAL_LIBS)
endif

5
src/config.c
View File

@ -821,7 +821,12 @@ void loadServerConfigFromString(char *config) {
if (err) goto loaderr;
}
} else if (!strcasecmp(argv[0],"scratch-file-path")) {
#ifdef USE_MEMKIND
storage_init(argv[1], server.maxmemory);
#else
err = "KeyDB not compliled with scratch-file support.";
goto loaderr;
#endif
} else {
err = "Bad directive or wrong number of arguments"; goto loaderr;
}

74
src/defrag.c
View File

@ -116,17 +116,17 @@ robj *activeDefragStringOb(robj* ob, long *defragged) {
/* try to defrag string object */
if (ob->type == OBJ_STRING) {
if(ob->encoding==OBJ_ENCODING_RAW) {
sds newsds = activeDefragSds((sds)ob->ptr);
sds newsds = activeDefragSds((sds)ptrFromObj(ob));
if (newsds) {
ob->ptr = newsds;
ob->m_ptr = newsds;
(*defragged)++;
}
} else if (ob->encoding==OBJ_ENCODING_EMBSTR) {
/* The sds is embedded in the object allocation, calculate the
* offset and update the pointer in the new allocation. */
long ofs = (intptr_t)ob->ptr - (intptr_t)ob;
long ofs = (intptr_t)ptrFromObj(ob) - (intptr_t)ob;
if ((ret = activeDefragAlloc(ob))) {
ret->ptr = (void*)((intptr_t)ret + ofs);
ret->m_ptr = (void*)((intptr_t)ret + ofs);
(*defragged)++;
}
} else if (ob->encoding!=OBJ_ENCODING_INT) {
@ -441,7 +441,7 @@ void defragLater(redisDb *db, dictEntry *kde) {
}
long scanLaterList(robj *ob) {
quicklist *ql = ob->ptr;
quicklist *ql = ptrFromObj(ob);
if (ob->type != OBJ_LIST || ob->encoding != OBJ_ENCODING_QUICKLIST)
return 0;
server.stat_active_defrag_scanned+=ql->len;
@ -463,7 +463,7 @@ void scanLaterZsetCallback(void *privdata, const dictEntry *_de) {
long scanLaterZset(robj *ob, unsigned long *cursor) {
if (ob->type != OBJ_ZSET || ob->encoding != OBJ_ENCODING_SKIPLIST)
return 0;
zset *zs = (zset*)ob->ptr;
zset *zs = (zset*)ptrFromObj(ob);
dict *d = zs->pdict;
scanLaterZsetData data = {zs, 0};
*cursor = dictScan(d, *cursor, scanLaterZsetCallback, defragDictBucketCallback, &data);
@ -483,7 +483,7 @@ long scanLaterSet(robj *ob, unsigned long *cursor) {
long defragged = 0;
if (ob->type != OBJ_SET || ob->encoding != OBJ_ENCODING_HT)
return 0;
dict *d = ob->ptr;
dict *d = ptrFromObj(ob);
*cursor = dictScan(d, *cursor, scanLaterSetCallback, defragDictBucketCallback, &defragged);
return defragged;
}
@ -504,7 +504,7 @@ long scanLaterHash(robj *ob, unsigned long *cursor) {
long defragged = 0;
if (ob->type != OBJ_HASH || ob->encoding != OBJ_ENCODING_HT)
return 0;
dict *d = ob->ptr;
dict *d = ptrFromObj(ob);
*cursor = dictScan(d, *cursor, scanLaterHashCallback, defragDictBucketCallback, &defragged);
return defragged;
}
@ -512,10 +512,10 @@ long scanLaterHash(robj *ob, unsigned long *cursor) {
long defragQuicklist(redisDb *db, dictEntry *kde) {
robj *ob = dictGetVal(kde);
long defragged = 0;
quicklist *ql = ob->ptr, *newql;
quicklist *ql = ptrFromObj(ob), *newql;
serverAssert(ob->type == OBJ_LIST && ob->encoding == OBJ_ENCODING_QUICKLIST);
if ((newql = activeDefragAlloc(ql)))
defragged++, ob->ptr = ql = newql;
defragged++, ob->m_ptr = ql = newql;
if (ql->len > server.active_defrag_max_scan_fields)
defragLater(db, kde);
else
@ -526,7 +526,7 @@ long defragQuicklist(redisDb *db, dictEntry *kde) {
long defragZsetSkiplist(redisDb *db, dictEntry *kde) {
robj *ob = dictGetVal(kde);
long defragged = 0;
zset *zs = (zset*)ob->ptr;
zset *zs = (zset*)ptrFromObj(ob);
zset *newzs;
zskiplist *newzsl;
dict *newdict;
@ -534,7 +534,7 @@ long defragZsetSkiplist(redisDb *db, dictEntry *kde) {
struct zskiplistNode *newheader;
serverAssert(ob->type == OBJ_ZSET && ob->encoding == OBJ_ENCODING_SKIPLIST);
if ((newzs = activeDefragAlloc(zs)))
defragged++, ob->ptr = zs = newzs;
defragged++, ob->m_ptr = zs = newzs;
if ((newzsl = activeDefragAlloc(zs->zsl)))
defragged++, zs->zsl = newzsl;
if ((newheader = activeDefragAlloc(zs->zsl->header)))
@ -561,16 +561,16 @@ long defragHash(redisDb *db, dictEntry *kde) {
robj *ob = dictGetVal(kde);
dict *d, *newd;
serverAssert(ob->type == OBJ_HASH && ob->encoding == OBJ_ENCODING_HT);
d = ob->ptr;
d = ptrFromObj(ob);
if (dictSize(d) > server.active_defrag_max_scan_fields)
defragLater(db, kde);
else
defragged += activeDefragSdsDict(d, DEFRAG_SDS_DICT_VAL_IS_SDS);
/* handle the dict struct */
if ((newd = activeDefragAlloc(ob->ptr)))
defragged++, ob->ptr = newd;
if ((newd = activeDefragAlloc(ptrFromObj(ob))))
defragged++, ob->m_ptr = newd;
/* defrag the dict tables */
defragged += dictDefragTables(ob->ptr);
defragged += dictDefragTables(ptrFromObj(ob));
return defragged;
}
@ -579,16 +579,16 @@ long defragSet(redisDb *db, dictEntry *kde) {
robj *ob = dictGetVal(kde);
dict *d, *newd;
serverAssert(ob->type == OBJ_SET && ob->encoding == OBJ_ENCODING_HT);
d = ob->ptr;
d = ptrFromObj(ob);
if (dictSize(d) > server.active_defrag_max_scan_fields)
defragLater(db, kde);
else
defragged += activeDefragSdsDict(d, DEFRAG_SDS_DICT_NO_VAL);
/* handle the dict struct */
if ((newd = activeDefragAlloc(ob->ptr)))
defragged++, ob->ptr = newd;
if ((newd = activeDefragAlloc(ptrFromObj(ob))))
defragged++, ob->m_ptr = newd;
/* defrag the dict tables */
defragged += dictDefragTables(ob->ptr);
defragged += dictDefragTables(ptrFromObj(ob));
return defragged;
}
@ -613,11 +613,11 @@ int scanLaterStraemListpacks(robj *ob, unsigned long *cursor, long long endtime,
return 0;
}
stream *s = ob->ptr;
raxStart(&ri,s->rax);
stream *s = ptrFromObj(ob);
raxStart(&ri,s->prax);
if (*cursor == 0) {
/* if cursor is 0, we start new iteration */
defragRaxNode(&s->rax->head);
defragRaxNode(&s->prax->head);
/* assign the iterator node callback before the seek, so that the
* initial nodes that are processed till the first item are covered */
ri.node_cb = defragRaxNode;
@ -738,19 +738,19 @@ long defragStream(redisDb *db, dictEntry *kde) {
long defragged = 0;
robj *ob = dictGetVal(kde);
serverAssert(ob->type == OBJ_STREAM && ob->encoding == OBJ_ENCODING_STREAM);
stream *s = ob->ptr, *news;
stream *s = ptrFromObj(ob), *news;
/* handle the main struct */
if ((news = activeDefragAlloc(s)))
defragged++, ob->ptr = s = news;
defragged++, ob->m_ptr = s = news;
if (raxSize(s->rax) > server.active_defrag_max_scan_fields) {
rax *newrax = activeDefragAlloc(s->rax);
if (raxSize(s->prax) > server.active_defrag_max_scan_fields) {
rax *newrax = activeDefragAlloc(s->prax);
if (newrax)
defragged++, s->rax = newrax;
defragged++, s->prax = newrax;
defragLater(db, kde);
} else
defragged += defragRadixTree(&s->rax, 1, NULL, NULL);
defragged += defragRadixTree(&s->prax, 1, NULL, NULL);
if (s->cgroups)
defragged += defragRadixTree(&s->cgroups, 1, defragStreamConsumerGroup, NULL);
@ -792,8 +792,8 @@ long defragKey(redisDb *db, dictEntry *de) {
if (ob->encoding == OBJ_ENCODING_QUICKLIST) {
defragged += defragQuicklist(db, de);
} else if (ob->encoding == OBJ_ENCODING_ZIPLIST) {
if ((newzl = activeDefragAlloc(ob->ptr)))
defragged++, ob->ptr = newzl;
if ((newzl = activeDefragAlloc(ptrFromObj(ob))))
defragged++, ob->m_ptr = newzl;
} else {
serverPanic("Unknown list encoding");
}
@ -801,16 +801,16 @@ long defragKey(redisDb *db, dictEntry *de) {
if (ob->encoding == OBJ_ENCODING_HT) {
defragged += defragSet(db, de);
} else if (ob->encoding == OBJ_ENCODING_INTSET) {
intset *newis, *is = ob->ptr;
intset *newis, *is = ptrFromObj(ob);
if ((newis = activeDefragAlloc(is)))
defragged++, ob->ptr = newis;
defragged++, ob->m_ptr = newis;
} else {
serverPanic("Unknown set encoding");
}
} else if (ob->type == OBJ_ZSET) {
if (ob->encoding == OBJ_ENCODING_ZIPLIST) {
if ((newzl = activeDefragAlloc(ob->ptr)))
defragged++, ob->ptr = newzl;
if ((newzl = activeDefragAlloc(ptrFromObj(ob))))
defragged++, ob->m_ptr = newzl;
} else if (ob->encoding == OBJ_ENCODING_SKIPLIST) {
defragged += defragZsetSkiplist(db, de);
} else {
@ -818,8 +818,8 @@ long defragKey(redisDb *db, dictEntry *de) {
}
} else if (ob->type == OBJ_HASH) {
if (ob->encoding == OBJ_ENCODING_ZIPLIST) {
if ((newzl = activeDefragAlloc(ob->ptr)))
defragged++, ob->ptr = newzl;
if ((newzl = activeDefragAlloc(ptrFromObj(ob))))
defragged++, ob->m_ptr = newzl;
} else if (ob->encoding == OBJ_ENCODING_HT) {
defragged += defragHash(db, de);
} else {

2
src/server.c
View File

@ -4851,7 +4851,9 @@ int main(int argc, char **argv) {
struct timeval tv;
int j;
#ifdef USE_MEMKIND
storage_init(NULL, 0);
#endif
#ifdef REDIS_TEST
if (argc == 3 && !strcasecmp(argv[1], "test")) {

6
src/storage.c
View File

@ -1,5 +1,7 @@
#include "server.h"
#ifdef USE_MEMKIND
#include <stdlib.h>
#include <stdio.h>
#include <memkind.h>
@ -264,4 +266,6 @@ void handle_postfork_child()
int fdOriginal = memkind_fd(mkdisk);
memkind_pmem_remapfd(mkdisk, fdNew);
close(fdOriginal);
}
}
#endif // USE_MEMKIND

24
src/zmalloc.c
View File

@ -63,14 +63,14 @@ void zlibc_free(void *ptr) {
#define realloc(ptr, size, type) srealloc(ptr, size, type)
#define free(ptr) sfree(ptr)
#elif defined(USE_TCMALLOC)
#define malloc(size) tc_malloc(size)
#define calloc(count,size) tc_calloc(count,size)
#define realloc(ptr,size) tc_realloc(ptr,size)
#define malloc(size, type) tc_malloc(size)
#define calloc(count,size, type) tc_calloc(count,size)
#define realloc(ptr,size, type) tc_realloc(ptr,size)
#define free(ptr) tc_free(ptr)
#elif defined(USE_JEMALLOC)
#define malloc(size) je_malloc(size)
#define calloc(count,size) je_calloc(count,size)
#define realloc(ptr,size) je_realloc(ptr,size)
#define malloc(size, type) je_malloc(size)
#define calloc(count,size,type) je_calloc(count,size)
#define realloc(ptr,size,type) je_realloc(ptr,size)
#define free(ptr) je_free(ptr)
#define mallocx(size,flags) je_mallocx(size,flags)
#define dallocx(ptr,flags) je_dallocx(ptr,flags)
@ -101,12 +101,8 @@ static void zmalloc_default_oom(size_t size) {
static void (*zmalloc_oom_handler)(size_t) = zmalloc_default_oom;
void *zmalloc(size_t size, enum MALLOC_CLASS class) {
#ifdef USE_MEMKIND
void *ptr = malloc(size+PREFIX_SIZE, class);
#else
(void)class;
void *ptr = malloc(size+PREFIX_SIZE);
#endif
void *ptr = malloc(size+PREFIX_SIZE, class);
if (!ptr) zmalloc_oom_handler(size);
#ifdef HAVE_MALLOC_SIZE
@ -138,12 +134,8 @@ void zfree_no_tcache(void *ptr) {
#endif
void *zcalloc(size_t size, enum MALLOC_CLASS class) {
#ifdef USE_MEMKIND
(void)(class);
void *ptr = calloc(1, size+PREFIX_SIZE, class);
#else
(void)class;
void *ptr = calloc(1, size+PREFIX_SIZE);
#endif
if (!ptr) zmalloc_oom_handler(size);
#ifdef HAVE_MALLOC_SIZE

1
src/zmalloc.h
View File

@ -36,7 +36,6 @@
#define __str(s) #s
#include "storage.h"
#define USE_MEMKIND 1
#if defined(USE_MEMKIND)
#define ZMALLOC_LIB ("memkind")
#undef USE_JEMALLOC