Performance enhancements!

Former-commit-id: 7175a8e4ebee74233702b8e308c0d57661b54bac

deps/hiredis/sds.h

@@ -85,7 +85,8 @@ struct __attribute__ ((__packed__)) sdshdr64 {
 
 static inline size_t sdslen(const sds s) {
     unsigned char flags = s[-1];
-    switch(flags&SDS_TYPE_MASK) {
+    
+    switch(__builtin_expect((flags&SDS_TYPE_MASK), SDS_TYPE_5)) {
         case SDS_TYPE_5:
             return SDS_TYPE_5_LEN(flags);
         case SDS_TYPE_8:

deps/memkind/src/src/memkind_pmem.c

@@ -74,6 +74,19 @@ MEMKIND_EXPORT void memkind_pmem_remapfd(struct memkind *kind, int fdNew)
 	priv->fd = fdNew;
 }
 
+MEMKIND_EXPORT int memkind_pmem_iskind(struct memkind *kind, void *pv)
+{
+    struct memkind_pmem *priv = kind->priv;
+    char *pb = pv;
+	for (int iextent = 0; iextent < priv->cextents; ++iextent)
+    {
+        struct memkind_pmem_extent *extent = priv->rgextents + iextent;
+        if ((((char*)extent->addrBase) <= pb) && (((char*)extent->addrBase)+extent->cb) > pb)
+            return 1;
+    }
+    return 0;
+}
+
 void *pmem_extent_alloc(extent_hooks_t *extent_hooks,
                         void *new_addr,
                         size_t size,

src/bio.c

@@ -86,7 +86,7 @@ struct bio_job {
 void *bioProcessBackgroundJobs(void *arg);
 void lazyfreeFreeObjectFromBioThread(robj *o);
 void lazyfreeFreeDatabaseFromBioThread(dict *ht1, dict *ht2);
-void lazyfreeFreeSlotsMapFromBioThread(zskiplist *sl);
+void lazyfreeFreeSlotsMapFromBioThread(rax *rt);
 
 /* Make sure we have enough stack to perform all the things we do in the
  * main thread. */

src/sds.h

@@ -92,17 +92,24 @@ struct __attribute__ ((__packed__)) sdshdr64 {
 
 static inline size_t sdslen(const sds s) {
     unsigned char flags = s[-1];
-    switch(flags&SDS_TYPE_MASK) {
+    int type = flags & SDS_TYPE_MASK;
-        case SDS_TYPE_5:
+
-            return SDS_TYPE_5_LEN(flags);
+    if (__builtin_expect((type == SDS_TYPE_5), 1))
-        case SDS_TYPE_8:
+    {
-            return SDS_HDR(8,s)->len;
+        return SDS_TYPE_5_LEN(flags);
-        case SDS_TYPE_16:
+    }
-            return SDS_HDR(16,s)->len;
+    else
-        case SDS_TYPE_32:
+    {
-            return SDS_HDR(32,s)->len;
+        switch(type) {
-        case SDS_TYPE_64:
+            case SDS_TYPE_8:
-            return SDS_HDR(64,s)->len;
+                return SDS_HDR(8,s)->len;
+            case SDS_TYPE_16:
+                return SDS_HDR(16,s)->len;
+            case SDS_TYPE_32:
+                return SDS_HDR(32,s)->len;
+            case SDS_TYPE_64:
+                return SDS_HDR(64,s)->len;
+        }
     }
     return 0;
 }

src/server.h

@@ -636,9 +636,9 @@ typedef struct redisObject {
     void *m_ptr;
 } robj;
 
-inline void *ptrFromObj(const robj *o)
+__attribute__((always_inline)) inline void *ptrFromObj(const robj *o)
 {
-    if (o->type == OBJ_STRING && o->encoding == OBJ_ENCODING_EMBSTR)
+    if (o->encoding == OBJ_ENCODING_EMBSTR)
         return ((char*)&((robj*)o)->m_ptr) + sizeof(struct sdshdr8);
     return o->m_ptr;
 }

src/storage.c

@@ -13,6 +13,8 @@
 struct memkind *mkdisk = NULL;
 static const char *PMEM_DIR = NULL;
 
+int memkind_pmem_iskind(struct memkind *kind, const void *pv);
+
 void handle_prefork();
 void handle_postfork_parent();
 void handle_postfork_child();
@@ -193,9 +195,19 @@ void sfree_objembstr(robj *obj)
     pool_free(&poolembstrobj, obj);
 }
 
+static memkind_t kindFromPtr(const void *pv)
+{
+    if (mkdisk == MEMKIND_DEFAULT)
+        return MEMKIND_DEFAULT;
+    
+    if (memkind_pmem_iskind(mkdisk, pv))
+        return mkdisk;
+    return MEMKIND_DEFAULT;
+}
+
 size_t salloc_usable_size(void *ptr)
 {
-    return memkind_malloc_usable_size(memkind_get_kind(ptr), ptr);
+    return memkind_malloc_usable_size(kindFromPtr(ptr), ptr);
 }
 
 static memkind_t kindFromClass(enum MALLOC_CLASS class)
@@ -225,7 +237,7 @@ void *scalloc(size_t cb, size_t c, enum MALLOC_CLASS class)
 
 void sfree(void *pv)
 {
-    memkind_free(NULL, pv);
+    memkind_free(kindFromPtr(pv), pv);
 }
 
 void *srealloc(void *pv, size_t cb, enum MALLOC_CLASS class)

src/zmalloc.c

@@ -175,7 +175,7 @@ void *zrealloc(void *ptr, size_t size, enum MALLOC_CLASS class) {
 #else
     realptr = (char*)ptr-PREFIX_SIZE;
     oldsize = *((size_t*)realptr);
-    newptr = realloc(realptr,size+PREFIX_SIZE);
+    newptr = realloc(realptr,size+PREFIX_SIZE, class);
     if (!newptr) zmalloc_oom_handler(size);
 
     *((size_t*)newptr) = size;

src/zmalloc.h

@@ -41,8 +41,9 @@
     #define ZMALLOC_LIB ("memkind")
     #undef USE_JEMALLOC
     #define USE_MALLOC_CLASS 1
-    #define HAVE_MALLOC_SIZE 1
+    // Even though memkind supports malloc_usable_size we don't use it for performance reasons
-    #define zmalloc_size(p) salloc_usable_size(p)
+    //#define HAVE_MALLOC_SIZE 0
+    //#define zmalloc_size(p) salloc_usable_size(p)
 #elif defined(USE_TCMALLOC)
 #define ZMALLOC_LIB ("tcmalloc-" __xstr(TC_VERSION_MAJOR) "." __xstr(TC_VERSION_MINOR))
 #include <google/tcmalloc.h>