From f1fc8edb77c5bf22a81380b6c2d9805d68c479f1 Mon Sep 17 00:00:00 2001 From: John Sully Date: Sat, 9 Feb 2019 13:05:06 -0500 Subject: [PATCH] Performance enhancements! Former-commit-id: 7175a8e4ebee74233702b8e308c0d57661b54bac --- deps/hiredis/sds.h | 3 ++- deps/memkind/src/src/memkind_pmem.c | 13 +++++++++++++ src/bio.c | 2 +- src/sds.h | 29 ++++++++++++++++++----------- src/server.h | 4 ++-- src/storage.c | 16 ++++++++++++++-- src/zmalloc.c | 2 +- src/zmalloc.h | 5 +++-- 8 files changed, 54 insertions(+), 20 deletions(-) diff --git a/deps/hiredis/sds.h b/deps/hiredis/sds.h index 13be75a9f..6b46297c2 100644 --- a/deps/hiredis/sds.h +++ b/deps/hiredis/sds.h @@ -85,7 +85,8 @@ struct __attribute__ ((__packed__)) sdshdr64 { static inline size_t sdslen(const sds s) { unsigned char flags = s[-1]; - switch(flags&SDS_TYPE_MASK) { + + switch(__builtin_expect((flags&SDS_TYPE_MASK), SDS_TYPE_5)) { case SDS_TYPE_5: return SDS_TYPE_5_LEN(flags); case SDS_TYPE_8: diff --git a/deps/memkind/src/src/memkind_pmem.c b/deps/memkind/src/src/memkind_pmem.c index b8caa0eea..a9e6d9089 100644 --- a/deps/memkind/src/src/memkind_pmem.c +++ b/deps/memkind/src/src/memkind_pmem.c @@ -74,6 +74,19 @@ MEMKIND_EXPORT void memkind_pmem_remapfd(struct memkind *kind, int fdNew) priv->fd = fdNew; } +MEMKIND_EXPORT int memkind_pmem_iskind(struct memkind *kind, void *pv) +{ + struct memkind_pmem *priv = kind->priv; + char *pb = pv; + for (int iextent = 0; iextent < priv->cextents; ++iextent) + { + struct memkind_pmem_extent *extent = priv->rgextents + iextent; + if ((((char*)extent->addrBase) <= pb) && (((char*)extent->addrBase)+extent->cb) > pb) + return 1; + } + return 0; +} + void *pmem_extent_alloc(extent_hooks_t *extent_hooks, void *new_addr, size_t size, diff --git a/src/bio.c b/src/bio.c index 13662197e..499a33054 100644 --- a/src/bio.c +++ b/src/bio.c @@ -86,7 +86,7 @@ struct bio_job { void *bioProcessBackgroundJobs(void *arg); void lazyfreeFreeObjectFromBioThread(robj *o); void lazyfreeFreeDatabaseFromBioThread(dict *ht1, dict *ht2); -void lazyfreeFreeSlotsMapFromBioThread(zskiplist *sl); +void lazyfreeFreeSlotsMapFromBioThread(rax *rt); /* Make sure we have enough stack to perform all the things we do in the * main thread. */ diff --git a/src/sds.h b/src/sds.h index 555a1d40c..3144dced8 100644 --- a/src/sds.h +++ b/src/sds.h @@ -92,17 +92,24 @@ struct __attribute__ ((__packed__)) sdshdr64 { static inline size_t sdslen(const sds s) { unsigned char flags = s[-1]; - switch(flags&SDS_TYPE_MASK) { - case SDS_TYPE_5: - return SDS_TYPE_5_LEN(flags); - case SDS_TYPE_8: - return SDS_HDR(8,s)->len; - case SDS_TYPE_16: - return SDS_HDR(16,s)->len; - case SDS_TYPE_32: - return SDS_HDR(32,s)->len; - case SDS_TYPE_64: - return SDS_HDR(64,s)->len; + int type = flags & SDS_TYPE_MASK; + + if (__builtin_expect((type == SDS_TYPE_5), 1)) + { + return SDS_TYPE_5_LEN(flags); + } + else + { + switch(type) { + case SDS_TYPE_8: + return SDS_HDR(8,s)->len; + case SDS_TYPE_16: + return SDS_HDR(16,s)->len; + case SDS_TYPE_32: + return SDS_HDR(32,s)->len; + case SDS_TYPE_64: + return SDS_HDR(64,s)->len; + } } return 0; } diff --git a/src/server.h b/src/server.h index f59b1d6f6..7d1f34e40 100644 --- a/src/server.h +++ b/src/server.h @@ -636,9 +636,9 @@ typedef struct redisObject { void *m_ptr; } robj; -inline void *ptrFromObj(const robj *o) +__attribute__((always_inline)) inline void *ptrFromObj(const robj *o) { - if (o->type == OBJ_STRING && o->encoding == OBJ_ENCODING_EMBSTR) + if (o->encoding == OBJ_ENCODING_EMBSTR) return ((char*)&((robj*)o)->m_ptr) + sizeof(struct sdshdr8); return o->m_ptr; } diff --git a/src/storage.c b/src/storage.c index 72472fd9f..584bbec27 100644 --- a/src/storage.c +++ b/src/storage.c @@ -13,6 +13,8 @@ struct memkind *mkdisk = NULL; static const char *PMEM_DIR = NULL; +int memkind_pmem_iskind(struct memkind *kind, const void *pv); + void handle_prefork(); void handle_postfork_parent(); void handle_postfork_child(); @@ -193,9 +195,19 @@ void sfree_objembstr(robj *obj) pool_free(&poolembstrobj, obj); } +static memkind_t kindFromPtr(const void *pv) +{ + if (mkdisk == MEMKIND_DEFAULT) + return MEMKIND_DEFAULT; + + if (memkind_pmem_iskind(mkdisk, pv)) + return mkdisk; + return MEMKIND_DEFAULT; +} + size_t salloc_usable_size(void *ptr) { - return memkind_malloc_usable_size(memkind_get_kind(ptr), ptr); + return memkind_malloc_usable_size(kindFromPtr(ptr), ptr); } static memkind_t kindFromClass(enum MALLOC_CLASS class) @@ -225,7 +237,7 @@ void *scalloc(size_t cb, size_t c, enum MALLOC_CLASS class) void sfree(void *pv) { - memkind_free(NULL, pv); + memkind_free(kindFromPtr(pv), pv); } void *srealloc(void *pv, size_t cb, enum MALLOC_CLASS class) diff --git a/src/zmalloc.c b/src/zmalloc.c index 35599ce0e..c081cb90f 100644 --- a/src/zmalloc.c +++ b/src/zmalloc.c @@ -175,7 +175,7 @@ void *zrealloc(void *ptr, size_t size, enum MALLOC_CLASS class) { #else realptr = (char*)ptr-PREFIX_SIZE; oldsize = *((size_t*)realptr); - newptr = realloc(realptr,size+PREFIX_SIZE); + newptr = realloc(realptr,size+PREFIX_SIZE, class); if (!newptr) zmalloc_oom_handler(size); *((size_t*)newptr) = size; diff --git a/src/zmalloc.h b/src/zmalloc.h index 979015c59..30ba80492 100644 --- a/src/zmalloc.h +++ b/src/zmalloc.h @@ -41,8 +41,9 @@ #define ZMALLOC_LIB ("memkind") #undef USE_JEMALLOC #define USE_MALLOC_CLASS 1 - #define HAVE_MALLOC_SIZE 1 - #define zmalloc_size(p) salloc_usable_size(p) + // Even though memkind supports malloc_usable_size we don't use it for performance reasons + //#define HAVE_MALLOC_SIZE 0 + //#define zmalloc_size(p) salloc_usable_size(p) #elif defined(USE_TCMALLOC) #define ZMALLOC_LIB ("tcmalloc-" __xstr(TC_VERSION_MAJOR) "." __xstr(TC_VERSION_MINOR)) #include